""" Same message, three languages: count ten request/reply exchanges (English, Traditional Chinese, Simplified Chinese) with two OpenAI tokenizers (tiktoken) and with each of four Gemini model ids (countTokens endpoint, free, no generation), then price a block of 1,000 such exchanges on the 13 chat models listed in pricing.json (Claude rows stay "not measured"). The corpus (60 passages) is the CORPUS constant below and is also written, in full, into the results file so it can be read and downloaded from there. Corpus rules * 10 exchanges, each with a request (what the user sends) and a reply (what the model answers), in English, Traditional Chinese and Simplified Chinese, matched sentence for sentence. The Simplified version is a re-write with mainland wording, not a glyph-only conversion; the wording swaps used are listed per exchange (zh_cn_term_swaps). * Plain factual requests only: no poetry, idioms or culture-bound terms. * Provenance: The sample conversations were drafted with AI assistance for this article and were not sent to any of the models being priced. The site owner reviewed three of the ten Traditional Chinese sets and edited one sentence (in both Chinese versions); the Simplified Chinese text has not been reviewed by a native speaker. No back-translation check and no professional translator. Reply token counts are therefore the counts of fixed texts, not generation volumes of any model under test. * Reply-to-request length ratio is a design parameter of this corpus (replies were drafted to be roughly 2-4 times as long as requests). Rules for the measurement (same as measure_gemini_vs_gpt_tokens.py) * Each Gemini id is called on its own; a failing id is recorded "n/a" and is never replaced by another id. * GPT side = tiktoken o200k_base (assumption for the GPT-5 family) and cl100k_base (reference only, not priced). * Gemini counts are net of the empty-string baseline (raw - baseline). * Claude is not measured and not estimated. Run: pip install tiktoken google-genai set GEMINI_API_KEY=... (your own key; never put it in a file) python measure_same_message_cost.py Options: --offline no API call; token counts and measured_on are read from the existing results file, pricing is re-read, everything else is recomputed and re-checked (tiktoken counts are re-encoded live and compared). --check-corpus only run the corpus checks and print a size table. --pricing PATH pricing.json (default: ../pricing.json next to this script) --out PATH results file (default: next to this script) Writes: measure_same_message_cost.results.json. Exit code 1 if any check fails (no results file is written in that case). """ import argparse import json import os import re import sys import time from datetime import date from decimal import Decimal, ROUND_HALF_UP from pathlib import Path import tiktoken HERE = Path(__file__).resolve().parent DEFAULT_PRICING = HERE.parent / "pricing.json" DEFAULT_OUT = HERE / "measure_same_message_cost.results.json" GPT_ENCODINGS = ["cl100k_base", "o200k_base"] GEMINI_MODELS = [ "gemini-3.1-pro-preview", "gemini-3.6-flash", "gemini-3.5-flash-lite", "gemini-3.1-flash-lite", ] LANGS = ["en", "zh_tw", "zh_cn"] ROLES = ["request", "reply"] RETRIES = 4 PAUSE_S = 0.15 EXCHANGES_PER_BLOCK = 1000 PAIR_COUNT = 10 # Rows used for the per-exchange table (both bill 2.00 in / 12.00 out, multiple 6) and # the worked example of the article: share of an English bill spent on output, and its size. PER_PAIR_ROWS = {"o200k": "gpt-5.6-terra", "gemini": "gemini/gemini-3.1-pro-preview"} EXAMPLE_OUT_SHARE = 0.6 EXAMPLE_ENGLISH_BILL_USD = 500 # Pooled chars/token measured by the earlier article (measure_gemini_vs_gpt_tokens, # measured_on 2026-10-13, its own 10-segment categories). Used only for the # cross-check; a difference is a register difference, not an error. G1_CPT = { "o200k_base": {"en": 5.224, "zh_tw": 1.181, "zh_cn": 1.401}, "gemini": {"en": 5.175, "zh_tw": 1.55, "zh_cn": 1.657}, } # Request length target (English, chars) and reply length target; checked. EN_REQUEST_RANGE = (150, 250) EN_REPLY_RANGE = (400, 650) # Full-width punctuation (U+FF00-FFEF) and CJK Ext-A are included on purpose. CJK_RE = re.compile("[　-〿㐀-䶿一-鿿＀-￯]") # -------------------------------------------------------------------------- # Corpus. Placeholders: none. "x" is an ASCII x in all three languages. # -------------------------------------------------------------------------- CORPUS = [ { "id": "p01", "topic": "polite email rewrite", "en": { "request": "Please rewrite this email so it sounds more polite: \"Send me the Q3 sales report by Friday. I need it for the meeting on Monday. Don't be late again.\" Keep it short.", "reply": "Here is a more polite version:\n\nSubject: Q3 sales report\n\nHi Dana,\n\nCould you please send me the Q3 sales report by Friday? I need it for the meeting on Monday morning. If I have it on time, I can prepare early, and I would be very grateful. If you expect any delay, please tell me as soon as possible so that we can adjust the schedule.\n\nThank you for your help.\n\nAlex\n\nI changed the commands into requests, gave a reason for the deadline, and replaced the reminder about being late with an offer to adjust the schedule.", }, "zh_tw": { "request": "請把這封電子郵件改寫得更有禮貌：「週五前把第三季銷售報告寄給我。週一的會議要用。不要再遲交了。」請保持簡短。", "reply": "以下是比較有禮貌的版本：\n\n主旨：第三季銷售報告\n\nDana 您好：\n\n可以請您在週五前把第三季銷售報告寄給我嗎？我週一上午的會議會用到。若能準時收到，我就能提早準備，我會非常感謝。如果您預計會有任何延誤，請盡快告訴我，我們可以調整時程。\n\n謝謝您的協助。\n\nAlex 敬上\n\n我把命令句改成了請求，為期限加上了理由，並把「不要再遲交」的提醒換成了調整時程的提議。", }, "zh_cn": { "request": "请把这封电子邮件改写得更有礼貌：“周五前把第三季度销售报告发给我。周一的会议要用。不要再迟交了。”请保持简短。", "reply": "以下是更有礼貌的版本：\n\n主题：第三季度销售报告\n\nDana 您好：\n\n可以请您在周五前把第三季度销售报告发给我吗？我周一上午的会议会用到。如果能按时收到，我就能提前准备，我会非常感谢。如果您预计会有任何延误，请尽快告诉我，我们可以调整日程。\n\n谢谢您的协助。\n\nAlex 敬上\n\n我把命令句改成了请求，为期限加上了理由，并把“不要再迟交”的提醒换成了调整日程的提议。", }, "zh_cn_term_swaps": [["比較有禮貌", "更有礼貌"], ["主旨", "主题"], ["寄給我", "发给我"], ["時程", "日程"]], }, { "id": "p02", "topic": "explain a text-editor setting", "en": { "request": "What does the 'Auto-save interval' option in my text editor do, and what value should I choose if my laptop battery is often low and I do not always carry a charger?", "reply": "The auto-save interval sets how often the editor writes your unsaved changes to disk. With a 1-minute interval, you lose at most one minute of work after a crash. With a 10-minute interval, you may lose up to ten minutes. Each save uses a little power, because the disk and the processor must wake up. For a laptop with a low battery, a value between 5 and 10 minutes is a reasonable balance. If your work is hard to redo, choose 2 or 3 minutes. You can also keep the interval short and plug in the charger when the battery drops below 20%.", }, "zh_tw": { "request": "我的文字編輯器裡有個「自動儲存間隔」選項，它是做什麼用的？如果我的筆電電池常常快沒電，而且我不一定隨身帶著充電器，應該選多少？", "reply": "自動儲存間隔決定編輯器多久會把尚未儲存的變更寫入磁碟一次。若間隔為 1 分鐘，當機後最多只會損失一分鐘的工作；若間隔為 10 分鐘，最多可能損失十分鐘。每次儲存都會消耗一點電力，因為磁碟與處理器必須被喚醒。對於電池電量偏低的筆電，5 到 10 分鐘之間是合理的折衷。如果你的工作很難重做，可以選 2 或 3 分鐘。你也可以維持較短的間隔，並在電量低於 20% 時接上充電器。", }, "zh_cn": { "request": "我的文本编辑器里有个“自动保存间隔”选项，它是做什么用的？如果我的笔记本电脑电池经常快没电，而且我不一定随身带着充电器，应该选多少？", "reply": "自动保存间隔决定编辑器多久会把尚未保存的更改写入磁盘一次。如果间隔为 1 分钟，崩溃后最多只会丢失一分钟的工作；如果间隔为 10 分钟，最多可能丢失十分钟。每次保存都会消耗一点电量，因为磁盘和处理器必须被唤醒。对于电池电量偏低的笔记本电脑，5 到 10 分钟之间是合理的折中。如果你的工作很难重做，可以选 2 或 3 分钟。你也可以保持较短的间隔，并在电量低于 20% 时接上充电器。", }, "zh_cn_term_swaps": [["儲存", "保存"], ["當機", "崩溃"], ["變更", "更改"], ["折衷", "折中"], ["筆電", "笔记本电脑"]], }, { "id": "p03", "topic": "three troubleshooting steps for slow Wi-Fi", "en": { "request": "My home Wi-Fi is very slow today. Can you give me three simple steps to try before I call my internet provider? My router is about four years old and I have not restarted it for months.", "reply": "Try these three steps in order.\n\n1. Restart the router. Unplug it for 30 seconds, plug it back in, and wait two minutes until all the lights are stable.\n2. Test the speed next to the router. If the speed is normal there but slow in other rooms, the signal is weak, so move the router to a central, open place.\n3. Check the other devices. Pause large downloads and video streams, because they share the same connection.\n\nIf the speed is still below half of your plan after these steps, call your provider and tell them what you tried.", }, "zh_tw": { "request": "我家的 Wi-Fi 今天非常慢。在打電話給網路業者之前，你可以給我三個簡單的步驟讓我先試試嗎？我的路由器大約有四年了，而且好幾個月沒有重新啟動過。", "reply": "請依序嘗試這三個步驟。\n\n1. 重新啟動路由器。拔掉電源 30 秒，再插回去，等兩分鐘直到所有指示燈穩定。\n2. 在路由器旁邊測試網速。如果在旁邊正常、在其他房間卻很慢，表示訊號太弱，請把路由器移到位置居中、沒有遮擋的地方。\n3. 檢查其他裝置。暫停大型下載與影片串流，因為它們共用同一條連線。\n\n如果做完這些步驟後，網速仍低於你方案的一半，就打電話給網路業者，並告訴他們你試過什麼。", }, "zh_cn": { "request": "我家的 Wi-Fi 今天非常慢。在打电话给网络运营商之前，你可以给我三个简单的步骤让我先试试吗？我的路由器大约有四年了，而且好几个月没有重启过。", "reply": "请按顺序尝试这三个步骤。\n\n1. 重启路由器。拔掉电源 30 秒，再插回去，等两分钟直到所有指示灯稳定。\n2. 在路由器旁边测试网速。如果在旁边正常、在其他房间却很慢，说明信号太弱，请把路由器移到位置居中、没有遮挡的地方。\n3. 检查其他设备。暂停大型下载和视频流，因为它们共用同一条连接。\n\n如果做完这些步骤后，网速仍低于你套餐的一半，就打电话给网络运营商，并告诉他们你试过什么。", }, "zh_cn_term_swaps": [["網路業者", "网络运营商"], ["重新啟動", "重启"], ["訊號", "信号"], ["裝置", "设备"], ["影片串流", "视频流"], ["方案", "套餐"]], }, { "id": "p04", "topic": "two-sentence product summary", "en": { "request": "Summarize this product description in two sentences: \"The AquaPure bottle holds 750 ml and keeps drinks cold for 24 hours or hot for 12 hours. It has a leak-proof lid, fits most car cup holders, and can go in the dishwasher.\"", "reply": "The AquaPure is a 750 ml insulated bottle that keeps drinks cold for 24 hours or hot for 12 hours. It has a leak-proof lid, fits most car cup holders, and can be washed in the dishwasher.\n\nIn short, it is a bottle for carrying drinks over a long day, and it is easy to use and to clean. Two details are not in the description: the material of the bottle and its weight when full. If those matter to you, check them before buying.", }, "zh_tw": { "request": "請用兩句話摘要這段產品說明：「AquaPure 水壺容量 750 毫升，可保冷 24 小時或保溫 12 小時。它的蓋子不會漏水，適合大多數車用杯架，也可以放進洗碗機清洗。」", "reply": "AquaPure 是一款容量 750 毫升的保溫水壺，可保冷 24 小時或保溫 12 小時。它的蓋子不會漏水，適合大多數車用杯架，也可以放進洗碗機清洗。\n\n簡單來說，這是一款適合長時間攜帶飲品的水壺，使用方便，也容易清洗。說明裡沒有提到兩個細節：水壺的材質，以及裝滿水時的重量。如果這兩點對你很重要，請在購買前先確認。", }, "zh_cn": { "request": "请用两句话概括这段产品说明：“AquaPure 水杯容量 750 毫升，可保冷 24 小时或保温 12 小时。它的盖子不会漏水，适合大多数车载杯架，也可以放进洗碗机清洗。”", "reply": "AquaPure 是一款容量 750 毫升的保温水杯，可保冷 24 小时或保温 12 小时。它的盖子不会漏水，适合大多数车载杯架，也可以放进洗碗机清洗。\n\n简单来说，这是一款适合长时间携带饮品的水杯，使用方便，也容易清洗。说明里没有提到两个细节：水杯的材质，以及装满水时的重量。如果这两点对你很重要，请在购买前先确认。", }, "zh_cn_term_swaps": [["摘要", "概括"], ["車用", "车载"], ["水壺", "水杯"]], }, { "id": "p05", "topic": "unit conversion with the calculation shown", "en": { "request": "I am driving 150 miles tomorrow and the weather forecast says 68 degrees Fahrenheit. How many kilometers is that, and how many degrees Celsius? Please list the calculation.", "reply": "150 miles is about 241 kilometers. The calculation is 150 x 1.609 = 241.35, so you can round it to 241 kilometers.\n\n68 degrees Fahrenheit is exactly 20 degrees Celsius. The calculation is (68 - 32) x 5 / 9 = 36 x 5 / 9 = 20.\n\nAt a steady speed of 100 kilometers per hour, the drive would take about two hours and 25 minutes, not counting any stops. A temperature of 20 degrees Celsius is mild, so a light jacket is enough for most people.", }, "zh_tw": { "request": "我明天要開車 150 英里，天氣預報說氣溫是華氏 68 度。這相當於多少公里？攝氏幾度？請列出計算過程。", "reply": "150 英里約為 241 公里。計算方式是 150 x 1.609 = 241.35，所以可以四捨五入為 241 公里。\n\n華氏 68 度恰好等於攝氏 20 度。計算方式是 (68 - 32) x 5 / 9 = 36 x 5 / 9 = 20。\n\n如果以每小時 100 公里的穩定速度行駛，不計中途停留，車程大約是兩小時 25 分鐘。攝氏 20 度的氣溫很溫和，對大多數人來說，只要穿一件薄外套就夠了。", }, "zh_cn": { "request": "我明天要开车 150 英里，天气预报说气温是华氏 68 度。这相当于多少公里？摄氏多少度？请写出计算过程。", "reply": "150 英里约为 241 公里。计算方法是 150 x 1.609 = 241.35，所以可以四舍五入为 241 公里。\n\n华氏 68 度恰好等于摄氏 20 度。计算方法是 (68 - 32) x 5 / 9 = 36 x 5 / 9 = 20。\n\n如果以每小时 100 公里的匀速行驶，不计中途停留，驾车时间大约是两小时 25 分钟。摄氏 20 度的气温很温和，对大多数人来说，只要穿一件薄外套就够了。", }, "zh_cn_term_swaps": [["列出", "写出"], ["計算方式", "计算方法"], ["穩定速度", "匀速"], ["車程", "驾车时间"]], }, { "id": "p06", "topic": "difference between two spreadsheet functions", "en": { "request": "In a spreadsheet, what is the difference between VLOOKUP and XLOOKUP? I only need a short answer for a beginner who uses a spreadsheet at work every day.", "reply": "Both functions find a value in a table and return a related value from the same row. VLOOKUP can only search the first column of the table and return a value from a column to the right, and you must count the column number yourself. XLOOKUP lets you choose the column to search and the column to return separately, so it can also look to the left. It finds an exact match by default and lets you set a message for a missing value. XLOOKUP is available in newer versions of the program, so an old version may only have VLOOKUP. For a beginner, XLOOKUP is easier to read and to fix.", }, "zh_tw": { "request": "在試算表裡，VLOOKUP 和 XLOOKUP 有什麼差別？我只需要給初學者的簡短答案，對方每天在工作中都會用到試算表。", "reply": "這兩個函數都能在表格中找到某個值，並從同一列傳回相關的值。VLOOKUP 只能搜尋表格的第一欄，並傳回右側某一欄的值，而且欄位編號要自己數。XLOOKUP 可以分別指定要搜尋的欄和要傳回的欄，所以也能往左邊找。它預設就是找完全相符的值，還可以為找不到的情況設定提示訊息。XLOOKUP 只有較新版本的程式才有，所以舊版本可能只有 VLOOKUP。對初學者來說，XLOOKUP 比較容易閱讀，也比較容易修正。", }, "zh_cn": { "request": "在电子表格里，VLOOKUP 和 XLOOKUP 有什么区别？我只需要给初学者的简短回答，对方每天在工作中都会用到电子表格。", "reply": "这两个函数都能在表格中找到某个值，并从同一行返回相关的值。VLOOKUP 只能搜索表格的第一列，并返回右侧某一列的值，而且列号要自己数。XLOOKUP 可以分别指定要搜索的列和要返回的列，所以也能往左边找。它默认就是查找完全匹配的值，还可以为找不到的情况设置提示信息。XLOOKUP 只有较新版本的软件才有，所以旧版本可能只有 VLOOKUP。对初学者来说，XLOOKUP 更容易阅读，也更容易修改。", }, "zh_cn_term_swaps": [["試算表", "电子表格"], ["差別", "区别"], ["答案", "回答"], ["同一列", "同一行"], ["第一欄", "第一列"], ["程式", "软件"], ["訊息", "信息"]], }, { "id": "p07", "topic": "meeting time across three cities", "en": { "request": "A team meeting starts at 9:00 a.m. on Tuesday in New York. New York is 5 hours behind London and 14 hours behind Tokyo. What time is the meeting in each of those two cities?", "reply": "The meeting is at 2:00 p.m. on Tuesday in London, because London is 5 hours ahead of New York: 9:00 + 5 hours = 14:00.\n\nIn Tokyo it is 11:00 p.m. on Tuesday, because Tokyo is 14 hours ahead: 9:00 + 14 hours = 23:00.\n\nThe Tokyo time is late in the evening, so the team there may find it inconvenient to join. You could move the meeting one hour earlier, to 8:00 a.m. in New York, which would be 10:00 p.m. in Tokyo, or you could record the meeting and share it afterwards.", }, "zh_tw": { "request": "團隊會議在紐約時間週二上午 9:00 開始。紐約比倫敦晚 5 小時，比東京晚 14 小時。這場會議在這兩個城市分別是幾點？", "reply": "會議在倫敦是週二下午 2:00，因為倫敦比紐約早 5 小時：9:00 + 5 小時 = 14:00。\n\n在東京則是週二晚上 11:00，因為東京早 14 小時：9:00 + 14 小時 = 23:00。\n\n東京的時間已是深夜，當地團隊可能不方便參加。你可以把會議提早一小時，也就是紐約上午 8:00，東京就是晚上 10:00；或者也可以錄下會議，事後再分享。", }, "zh_cn": { "request": "团队会议在纽约时间周二上午 9:00 开始。纽约比伦敦晚 5 小时，比东京晚 14 小时。这场会议在这两个城市分别是几点？", "reply": "会议在伦敦是周二下午 2:00，因为伦敦比纽约早 5 小时：9:00 + 5 小时 = 14:00。\n\n在东京则是周二晚上 11:00，因为东京早 14 小时：9:00 + 14 小时 = 23:00。\n\n东京的时间已是深夜，当地团队可能难以参加。你可以把会议提前一小时，也就是纽约上午 8:00，东京就是晚上 10:00；或者也可以录制会议，事后再分享。", }, "zh_cn_term_swaps": [["提早", "提前"], ["錄下", "录制"], ["不方便參加", "难以参加"]], }, { "id": "p08", "topic": "scale a recipe from 4 to 6 servings", "en": { "request": "My pancake recipe makes 4 servings and uses 200 g of flour, 300 ml of milk and 2 eggs. How much of each do I need for 6 servings? I want to cook for six people tomorrow morning.", "reply": "For 6 servings, multiply every amount by 1.5.\n\nFlour: 200 g x 1.5 = 300 g.\nMilk: 300 ml x 1.5 = 450 ml.\nEggs: 2 x 1.5 = 3 eggs.\n\nThe numbers are whole, so no rounding is needed. Prepare the batter in the same way as in the original recipe and let it rest for five minutes. With a medium pan you will probably cook the pancakes in two or three rounds, so keep the finished ones warm in a low oven at about 80 degrees Celsius.", }, "zh_tw": { "request": "我的鬆餅食譜可做 4 人份，用 200 公克麵粉、300 毫升牛奶和 2 顆雞蛋。做 6 人份的話，每樣材料各需要多少？我明天早上要煮給六個人吃。", "reply": "要做 6 人份，每樣材料都乘以 1.5。\n\n麵粉：200 公克 x 1.5 = 300 公克。\n牛奶：300 毫升 x 1.5 = 450 毫升。\n雞蛋：2 顆 x 1.5 = 3 顆。\n\n數字都是整數，所以不需要四捨五入。麵糊的調製方式與原本的食譜相同，調好後靜置五分鐘。用中型平底鍋的話，大概要分兩到三輪煎，所以煎好的鬆餅可以放進攝氏 80 度左右的低溫烤箱保溫。", }, "zh_cn": { "request": "我的松饼食谱可做 4 人份，用 200 克面粉、300 毫升牛奶和 2 个鸡蛋。做 6 人份的话，每样材料各需要多少？我明天早上要做给六个人吃。", "reply": "要做 6 人份，每样材料都乘以 1.5。\n\n面粉：200 克 x 1.5 = 300 克。\n牛奶：300 毫升 x 1.5 = 450 毫升。\n鸡蛋：2 个 x 1.5 = 3 个。\n\n数字都是整数，所以不需要四舍五入。面糊的调制方法与原来的食谱相同，调好后静置五分钟。用中号平底锅的话，大概要分两到三轮煎，所以煎好的松饼可以放进摄氏 80 度左右的低温烤箱保温。", }, "zh_cn_term_swaps": [["200 公克", "200 克"], ["顆", "个"], ["煮給", "做给"], ["調製方式", "调制方法"]], }, { "id": "p09", "topic": "explain a disk-space error message", "en": { "request": "My computer says \"Not enough disk space to complete the operation\" when I try to install an update. What does it mean and what should I do? I have about 2 GB free.", "reply": "The message means that the drive has less free space than the update needs. Updates need room both for the download and for the files that are unpacked during the installation, so the free space must be larger than the size of the update itself. With only 2 GB free, a large update will fail.\n\nTo fix it, first empty the recycle bin and delete files you no longer need, such as old downloads. Next, uninstall programs that you do not use. Then move large photos or videos to an external drive or to cloud storage. When at least 10 GB are free, try the update again.", }, "zh_tw": { "request": "我想安裝更新時，電腦顯示「磁碟空間不足，無法完成操作」。這是什麼意思，我該怎麼做？我大約還有 2 GB 的可用空間。", "reply": "這個訊息表示磁碟的剩餘空間比更新所需的還少。更新既需要空間存放下載的檔案，也需要空間存放安裝過程中解開的檔案，所以剩餘空間必須比更新本身的大小還大。只剩 2 GB 的話，較大的更新就會失敗。\n\n要解決這個問題，首先清空資源回收筒，並刪除你不再需要的檔案，例如舊的下載檔。接著解除安裝你沒在用的程式。然後把大型的相片或影片移到外接硬碟或雲端儲存空間。當至少有 10 GB 的可用空間時，再試一次更新。", }, "zh_cn": { "request": "我想安装更新时，电脑显示“磁盘空间不足，无法完成操作”。这是什么意思，我该怎么办？我大约还有 2 GB 的可用空间。", "reply": "这条消息表示磁盘的剩余空间比更新所需的还少。更新既需要空间存放下载的文件，也需要空间存放安装过程中解压的文件，所以剩余空间必须比更新本身的大小还大。只剩 2 GB 的话，较大的更新就会失败。\n\n要解决这个问题，首先清空回收站，并删除你不再需要的文件，例如旧的下载文件。接着卸载你没在用的程序。然后把大型的照片或视频移到外接硬盘或云存储空间。当至少有 10 GB 的可用空间时，再试一次更新。", }, "zh_cn_term_swaps": [["該怎麼做", "该怎么办"], ["資源回收筒", "回收站"], ["檔案", "文件"], ["解除安裝", "卸载"], ["程式", "程序"], ["影片", "视频"]], }, { "id": "p10", "topic": "can headphones still be returned", "en": { "request": "I bought a pair of headphones 12 days ago. The store's policy says returns are accepted within 14 days if the box is unopened. I already opened the box. Can I still return them?", "reply": "Based on the policy you quoted, probably not as a normal return. The first condition is met, because 12 days is within the 14-day period. The second condition is not met, because the box has been opened, and the policy says the box must be unopened.\n\nThere are still some options. You can contact the store, explain the situation, and ask whether they make an exception. Some stores do this for unused products. If the headphones are faulty, the policy for defects is usually separate from the return policy, so ask about that too. Keep your receipt and the box in case you need them.", }, "zh_tw": { "request": "我 12 天前買了一副耳機。商店的規定是，在 14 天內、盒子未拆封的情況下可以退貨。我已經拆開盒子了。我還能退貨嗎？", "reply": "依照你引述的規定，這樣大概無法以一般方式退貨。第一個條件符合，因為 12 天在 14 天的期限內。第二個條件不符合，因為盒子已經拆開，而規定要求盒子必須未拆封。\n\n不過你還有一些選擇。你可以聯絡商店，說明情況，並詢問他們是否願意破例。有些商店會對未使用的商品這麼做。如果耳機有瑕疵，瑕疵商品的處理規定通常與退貨規定分開，所以也要問一下。請保留收據和盒子，以備不時之需。", }, "zh_cn": { "request": "我 12 天前买了一副耳机。商店的规定是，在 14 天内、盒子未拆封的情况下可以退货。我已经拆开盒子了。我还能退货吗？", "reply": "按照你引用的规定，这样大概无法按普通方式退货。第一个条件符合，因为 12 天在 14 天的期限内。第二个条件不符合，因为盒子已经拆开，而规定要求盒子必须未拆封。\n\n不过你还有一些选择。你可以联系商店，说明情况，并询问他们是否愿意破例。有些商店会对未使用的商品这样做。如果耳机有缺陷，缺陷商品的处理规定通常与退货规定分开，所以也要问一下。请保留收据和盒子，以备不时之需。", }, "zh_cn_term_swaps": [["依照", "按照"], ["聯絡", "联系"], ["瑕疵", "缺陷"], ["一般方式", "普通方式"]], }, ] # -------------------------------------------------------------------------- # Helpers # -------------------------------------------------------------------------- def segkey(pid, lang, role): return pid + "|" + lang + "|" + role def iter_segments(corpus): for pair in corpus: for lang in LANGS: for role in ROLES: yield pair["id"], lang, role, pair[lang][role] def has_cjk(text): return CJK_RE.search(text) is not None def corpus_checks(corpus): """Return a list of failure strings (empty list = all passed).""" bad = [] if len(corpus) != PAIR_COUNT: bad.append("pair count %d != %d" % (len(corpus), PAIR_COUNT)) ids = [p["id"] for p in corpus] if len(set(ids)) != len(ids): bad.append("duplicate pair ids") for pair in corpus: pid = pair["id"] for lang in LANGS: for role in ROLES: t = pair.get(lang, {}).get(role, "") if not isinstance(t, str) or not t.strip(): bad.append("%s %s %s empty" % (pid, lang, role)) continue if lang == "en" and has_cjk(t): bad.append("%s en %s contains CJK" % (pid, role)) if lang != "en" and not has_cjk(t): bad.append("%s %s %s has no CJK" % (pid, lang, role)) if bad and bad[-1].startswith(pid + " "): continue for role in ROLES: if pair["zh_tw"][role] == pair["zh_cn"][role]: bad.append("%s zh_tw == zh_cn (%s)" % (pid, role)) if len(pair["en"]["reply"]) <= len(pair["en"]["request"]): bad.append("%s en reply not longer than request" % pid) lo, hi = EN_REQUEST_RANGE if not lo <= len(pair["en"]["request"]) <= hi: bad.append("%s en request %d chars outside %d-%d" % (pid, len(pair["en"]["request"]), lo, hi)) lo, hi = EN_REPLY_RANGE if not lo <= len(pair["en"]["reply"]) <= hi: bad.append("%s en reply %d chars outside %d-%d" % (pid, len(pair["en"]["reply"]), lo, hi)) # Simplified is a re-write with mainland wording, not a glyph-only conversion. swaps = pair.get("zh_cn_term_swaps", []) if len(swaps) < 3: bad.append("%s fewer than 3 zh_cn term swaps" % pid) tw_all = pair["zh_tw"]["request"] + "\n" + pair["zh_tw"]["reply"] cn_all = pair["zh_cn"]["request"] + "\n" + pair["zh_cn"]["reply"] for tw_term, cn_term in swaps: if tw_term not in tw_all: bad.append("%s swap tw term %r not in zh_tw" % (pid, tw_term)) if cn_term not in cn_all: bad.append("%s swap cn term %r not in zh_cn" % (pid, cn_term)) if tw_term in cn_all: bad.append("%s swap tw term %r still in zh_cn" % (pid, tw_term)) if cn_term in tw_all: bad.append("%s swap cn term %r found in zh_tw" % (pid, cn_term)) return bad def count_gemini(client, model, text): last = None for attempt in range(RETRIES): try: resp = client.models.count_tokens(model=model, contents=text) return int(resp.total_tokens) except Exception as exc: # network / quota / model errors last = exc time.sleep(1.5 * (attempt + 1)) raise RuntimeError(type(last).__name__ + ": " + str(last)[:200]) def measure_online(corpus): key = os.environ.get("GEMINI_API_KEY") if not key: print("GEMINI_API_KEY is not set; cannot call countTokens (use --offline to reuse stored counts).") raise SystemExit(1) from google import genai client = genai.Client(api_key=key) status, raw = {}, {} for gid in GEMINI_MODELS: s = {"requested_model_id": gid} try: baseline = count_gemini(client, gid, "") counts = {} for pid, lang, role, text in iter_segments(corpus): counts[segkey(pid, lang, role)] = count_gemini(client, gid, text) time.sleep(PAUSE_S) except Exception as exc: s["status"] = "n/a" s["error"] = str(exc).replace(key, "").encode("ascii", "replace").decode() else: s["status"] = "ok" s["baseline_tokens_empty_string"] = baseline raw[gid] = counts status[gid] = s return {"measured_on": date.today().isoformat(), "gemini_model_status": status, "gemini_raw": raw} def gpt_counts(corpus): enc = {n: tiktoken.get_encoding(n) for n in GPT_ENCODINGS} return {segkey(pid, lang, role): {n: len(enc[n].encode(text)) for n in GPT_ENCODINGS} for pid, lang, role, text in iter_segments(corpus)} def load_counts_offline(path, corpus): """Read stored counts. The stored text must equal the corpus text.""" old = json.loads(Path(path).read_text(encoding="utf-8")) status = old["gemini_model_status"] raw = {g: {} for g in GEMINI_MODELS if status[g]["status"] == "ok"} gpt = {} by_id = {p["id"]: p for p in old["corpus"]["pairs"]} for pid, lang, role, text in iter_segments(corpus): seg = by_id[pid]["per_lang"][lang][role] if seg["text"] != text: print("offline: corpus text of %s %s %s differs from the stored one; re-measure online." % (pid, lang, role)) raise SystemExit(1) sk = segkey(pid, lang, role) gpt[sk] = {n: seg["tokens"][n] for n in GPT_ENCODINGS} for g in raw: raw[g][sk] = seg["tokens_raw_with_overhead"][g] print("offline: gemini counts reused from %s" % old["measured_on"]) return {"measured_on": old["measured_on"], "gemini_model_status": status, "gemini_raw": raw, "gpt_tokens": gpt} # -------------------------------------------------------------------------- # Results # -------------------------------------------------------------------------- def basis_tokenizer_for(row, ok_ids): """Which tokenizer's counts price this model row (None = not priced).""" if row["vendor"] == "OpenAI": return "o200k_base" if row["vendor"] == "Google" and row["id"].startswith("gemini/") and row["id"][len("gemini/"):] in ok_ids: return row["id"][len("gemini/"):] return None def hu(x, places=0): """Round half up (Decimal) on the shortest decimal form of x, so a value that ends in exactly 5 goes up. Python's round() and "%.Nf" decide such ties by the float's binary expansion and are not uniform.""" d = x if isinstance(x, Decimal) else Decimal(repr(x)) return d.quantize(Decimal(1).scaleb(-places), rounding=ROUND_HALF_UP) def r3(x): return float(hu(x, 3)) def r4(x): return float(hu(x, 4)) def exact_cost_1000(tin, tout, rate_in, rate_out): """Exact decimal price of 1000 exchanges from the summed token counts (ten exchanges x 100).""" c = (Decimal(tin) * Decimal(repr(rate_in)) + Decimal(tout) * Decimal(repr(rate_out))) / Decimal(10 ** 6) return c * EXCHANGES_PER_BLOCK / PAIR_COUNT def resolve(results, path): cur = results for part in path: if isinstance(cur, list): if isinstance(part, int): cur = cur[part] else: cur = next(item for item in cur if item["id"] == part) else: cur = cur[part] return cur def build_results(counts, pricing, corpus): status = counts["gemini_model_status"] ok_ids = [g for g in GEMINI_MODELS if status[g]["status"] == "ok"] tok_labels = GPT_ENCODINGS + ok_ids gpt = counts["gpt_tokens"] raw = counts["gemini_raw"] pairs = [] for pair in corpus: per_lang = {} for lang in LANGS: entry = {} for role in ROLES: sk = segkey(pair["id"], lang, role) text = pair[lang][role] tokens = dict(gpt[sk]) for g in ok_ids: tokens[g] = raw[g][sk] - status[g]["baseline_tokens_empty_string"] entry[role] = { "text": text, "chars": len(text), "tokens": tokens, "tokens_raw_with_overhead": {g: raw[g][sk] for g in ok_ids}, } per_lang[lang] = entry pairs.append({"id": pair["id"], "topic": pair["topic"], "per_lang": per_lang, "zh_cn_term_swaps": pair["zh_cn_term_swaps"]}) # totals --------------------------------------------------------------- sums = {} # lang -> {chars_request, chars_reply, tok:{request, reply, raw}} for lang in LANGS: s = {"chars_request": 0, "chars_reply": 0, "tok": {t: {"request": 0, "reply": 0, "raw": 0} for t in tok_labels}} for p in pairs: for role in ROLES: seg = p["per_lang"][lang][role] s["chars_" + role] += seg["chars"] for t in tok_labels: s["tok"][t][role] += seg["tokens"][t] for g in ok_ids: s["tok"][g]["raw"] += seg["tokens_raw_with_overhead"][g] sums[lang] = s def tot(lang, t): return sums[lang]["tok"][t]["request"] + sums[lang]["tok"][t]["reply"] totals = {} for lang in LANGS: s = sums[lang] chars_total = s["chars_request"] + s["chars_reply"] t_out = {} for t in tok_labels: e = {"request": s["tok"][t]["request"], "reply": s["tok"][t]["reply"], "total": tot(lang, t)} if t in ok_ids: e["raw_with_overhead"] = s["tok"][t]["raw"] t_out[t] = e totals[lang] = { "chars_request": s["chars_request"], "chars_reply": s["chars_reply"], "chars_total": chars_total, "tokens": t_out, "chars_ratio_vs_en": r3(chars_total / (sums["en"]["chars_request"] + sums["en"]["chars_reply"])), "tokens_ratio_vs_en": {t: r3(tot(lang, t) / tot("en", t)) for t in tok_labels}, "tokens_ratio_vs_en_request": {t: r3(s["tok"][t]["request"] / sums["en"]["tok"][t]["request"]) for t in tok_labels}, "tokens_ratio_vs_en_reply": {t: r3(s["tok"][t]["reply"] / sums["en"]["tok"][t]["reply"]) for t in tok_labels}, "pooled_chars_per_token": {t: r3(chars_total / tot(lang, t)) for t in tok_labels}, } # models --------------------------------------------------------------- models = [] unr = {} # model id -> lang -> unrounded numbers for row in pricing["chat"]: tok = basis_tokenizer_for(row, ok_ids) m = {"id": row["id"], "name": row["name"], "vendor": row["vendor"], "in": row["in"], "out": row["out"], "out_over_in": r3(row["out"] / row["in"])} if tok is None: m.update({"basis_tokenizer": None, "basis": "n/a", "status": "not measured", "per_lang": None}) models.append(m) continue m["basis_tokenizer"] = tok m["basis"] = "o200k_base (assumed)" if tok == "o200k_base" else tok + " (measured)" m["status"] = "ok" per_lang, u = {}, {} base_cost = None for lang in LANGS: tin = sums[lang]["tok"][tok]["request"] tout = sums[lang]["tok"][tok]["reply"] c_in, c_out = tin * row["in"] / 1e6, tout * row["out"] / 1e6 cost = (c_in + c_out) * EXCHANGES_PER_BLOCK / PAIR_COUNT if lang == "en": base_cost = cost cost_x = exact_cost_1000(tin, tout, row["in"], row["out"]) u[lang] = {"cost": cost, "cost_exact": cost_x, "ratio": cost / base_cost, "out_share": c_out / (c_in + c_out)} per_lang[lang] = {"tokens_in": tin, "tokens_out": tout, "cost_1000_usd": float(cost_x.quantize(Decimal("0.0001"), rounding=ROUND_HALF_UP)), "ratio_vs_en": r3(cost / base_cost), "out_share": r3(c_out / (c_in + c_out)), "out_share_exact": float(hu(c_out / (c_in + c_out), 6))} m["per_lang"] = per_lang unr[row["id"]] = u models.append(m) measured = [m for m in models if m["per_lang"] is not None] groups = {"o200k": [m for m in measured if m["basis_tokenizer"] == "o200k_base"], "gemini": [m for m in measured if m["basis_tokenizer"] != "o200k_base"]} # summary -------------------------------------------------------------- cost_ratio_range = {} for gname, rows in groups.items(): cost_ratio_range[gname] = {} for lang in ("zh_tw", "zh_cn"): if not rows: continue lo = min(rows, key=lambda m: unr[m["id"]][lang]["ratio"]) hi = max(rows, key=lambda m: unr[m["id"]][lang]["ratio"]) cost_ratio_range[gname][lang] = { "min": r3(unr[lo["id"]][lang]["ratio"]), "min4": r4(unr[lo["id"]][lang]["ratio"]), "min_id": lo["id"], "max": r3(unr[hi["id"]][lang]["ratio"]), "max4": r4(unr[hi["id"]][lang]["ratio"]), "max_id": hi["id"]} cheapest, priciest, out_share_range = {}, {}, {} for lang in LANGS: lo = min(measured, key=lambda m: unr[m["id"]][lang]["cost"]) hi = max(measured, key=lambda m: unr[m["id"]][lang]["cost"]) cheapest[lang] = {"id": lo["id"], "cost": float(unr[lo["id"]][lang]["cost_exact"].quantize(Decimal("0.0001"), rounding=ROUND_HALF_UP))} priciest[lang] = {"id": hi["id"], "cost": float(unr[hi["id"]][lang]["cost_exact"].quantize(Decimal("0.0001"), rounding=ROUND_HALF_UP))} slo = min(measured, key=lambda m: unr[m["id"]][lang]["out_share"]) shi = max(measured, key=lambda m: unr[m["id"]][lang]["out_share"]) out_share_range[lang] = { "min": {"id": slo["id"], "share": r3(unr[slo["id"]][lang]["out_share"]), "share_exact": float(hu(unr[slo["id"]][lang]["out_share"], 6))}, "max": {"id": shi["id"], "share": r3(unr[shi["id"]][lang]["out_share"]), "share_exact": float(hu(unr[shi["id"]][lang]["out_share"], 6))}} ks = [m["out"] / m["in"] for m in measured] eq_lo, eq_hi = min(k / (1 + k) for k in ks), max(k / (1 + k) for k in ks) reply_to_request_en = sums["en"]["tok"]["o200k_base"]["reply"] / sums["en"]["tok"]["o200k_base"]["request"] # per exchange: token ratios and priced ratios (Chinese / English), unrounded values kept for min/max by_id = {m["id"]: m for m in measured} per_pair, pair_unr = [], {} for p in pairs: def tk(lang, role, t, _p=p): return _p["per_lang"][lang][role]["tokens"][t] tokr, priced, pu = {}, {}, {} for t in ["o200k_base"] + ok_ids: tokr[t] = {} for lang in ("zh_tw", "zh_cn"): e = {role: r3(tk(lang, role, t) / tk("en", role, t)) for role in ROLES} e["total"] = r3(sum(tk(lang, r, t) for r in ROLES) / sum(tk("en", r, t) for r in ROLES)) tokr[t][lang] = e for m in measured: t = m["basis_tokenizer"] cost = lambda lang, _m=m, _t=t: tk(lang, "request", _t) * _m["in"] + tk(lang, "reply", _t) * _m["out"] pu[m["id"]] = {lang: cost(lang) / cost("en") for lang in ("zh_tw", "zh_cn")} priced[m["id"]] = {lang: r3(v) for lang, v in pu[m["id"]].items()} pair_unr[p["id"]] = pu per_pair.append({"id": p["id"], "topic": p["topic"], "tokens_ratio_vs_en": tokr, "priced_ratio_vs_en": priced}) per_pair_range = {} for g, rid in PER_PAIR_ROWS.items(): if rid not in by_id: continue per_pair_range[g] = {"row_id": rid} for lang in ("zh_tw", "zh_cn"): vals = [(pair_unr[p["id"]][rid][lang], p["id"], p["topic"]) for p in pairs] lo, hi = min(vals), max(vals) per_pair_range[g][lang] = { "min": r3(lo[0]), "min_id": lo[1], "min_topic": lo[2], "max": r3(hi[0]), "max_id": hi[1], "max_topic": hi[2], "below_english_topics": [t for v, i, t in vals if r3(v) < 1]} # worked example: a bill whose output share is EXAMPLE_OUT_SHARE (weights applied to r_in / r_out) example_mix = {"out_share_of_english_bill": EXAMPLE_OUT_SHARE, "in_share_of_english_bill": float(hu(1 - EXAMPLE_OUT_SHARE, 6)), "english_bill_usd": EXAMPLE_ENGLISH_BILL_USD} for g, tok in (("o200k", "o200k_base"), ("gemini", ok_ids[0] if ok_ids else None)): if tok is None: continue example_mix[g] = {"basis_tokenizer": tok} for lang in ("zh_tw", "zh_cn"): r_in = sums[lang]["tok"][tok]["request"] / sums["en"]["tok"][tok]["request"] r_out = sums[lang]["tok"][tok]["reply"] / sums["en"]["tok"][tok]["reply"] ratio = (1 - EXAMPLE_OUT_SHARE) * r_in + EXAMPLE_OUT_SHARE * r_out example_mix[g][lang] = {"ratio": r3(ratio), "bill_usd": int(hu(EXAMPLE_ENGLISH_BILL_USD * ratio))} cross = {"g1_source": "measure_gemini_vs_gpt_tokens.results.json (measured_on 2026-10-13; its own samples)", "o200k_base": {}, "gemini": {}} for lang in LANGS: cross["o200k_base"][lang] = {"this": totals[lang]["pooled_chars_per_token"]["o200k_base"], "g1": G1_CPT["o200k_base"][lang]} cross["gemini"][lang] = {"this": {g: totals[lang]["pooled_chars_per_token"][g] for g in ok_ids}, "g1": G1_CPT["gemini"][lang]} summary = { "chars_ratio": {l: totals[l]["chars_ratio_vs_en"] for l in ("zh_tw", "zh_cn")}, "tokens_ratio": {t: {l: totals[l]["tokens_ratio_vs_en"][t] for l in ("zh_tw", "zh_cn")} for t in tok_labels}, "tokens_ratio_request": {t: {l: totals[l]["tokens_ratio_vs_en_request"][t] for l in ("zh_tw", "zh_cn")} for t in tok_labels}, "tokens_ratio_reply": {t: {l: totals[l]["tokens_ratio_vs_en_reply"][t] for l in ("zh_tw", "zh_cn")} for t in tok_labels}, "cost_ratio_range": cost_ratio_range, "cheapest_1000": cheapest, "priciest_1000": priciest, "out_share_range": out_share_range, "reply_to_request_tokens_en_o200k": r3(reply_to_request_en), "out_share_if_equal_length": {"min": r3(eq_lo), "max": r3(eq_hi)}, "per_pair_priced_range": per_pair_range, "example_mix": example_mix, "crosscheck_vs_g1": cross, } results = { "measured_on": counts["measured_on"], "pricing_source": pricing.get("source"), "pricing_last_synced": pricing.get("last_synced"), "gpt_encodings_measured": GPT_ENCODINGS, "gemini_models_requested": GEMINI_MODELS, "gemini_models_measured": ok_ids, "gemini_model_status": status, "gemini_response_model_id_note": ( "countTokens returns only total_tokens; the response carries no model id, " "so the requested id is the only id recorded."), "claude_measured": False, "claude_note": "Claude models: not measured, not estimated (no key for its counter).", "exchanges_per_block": EXCHANGES_PER_BLOCK, "corpus": { "pair_count": PAIR_COUNT, "languages": LANGS, "equivalence_note": ( "zh_tw and zh_cn match en sentence for sentence " "(same facts, same numbers, same units); zh_cn is a re-write with mainland wording, " "not a glyph-only conversion. No back-translation and no professional translator."), "provenance_note": ( "The sample conversations were drafted with AI assistance for this article and were not " "sent to any of the models being priced. The site owner reviewed three of the ten " "Traditional Chinese sets and edited one sentence (in both Chinese versions); the " "Simplified Chinese text has not been reviewed by a native speaker."), "design_parameters": {"en_request_chars": list(EN_REQUEST_RANGE), "en_reply_chars": list(EN_REPLY_RANGE)}, "pairs": pairs, }, "per_pair": per_pair, "totals": totals, "models": models, "summary": summary, } results["display"], results["display_sources"] = build_display(results) return results def build_display(results): """Every number the article may quote, as a final string, plus its source path.""" disp, src = {}, {} def put(key, text, path): disp[key] = text src[key] = path money = lambda v: "$%.4f" % v ratio = lambda v: "%.3f×" % v ratio4 = lambda v: "%.4f×" % v mult = lambda v: "%g×" % v usd0 = lambda v: "$" + format(int(v), ",") pct = lambda v: "%d%%" % hu(Decimal(repr(v)) * 100) grp = lambda n: format(n, ",") tot = results["totals"] for lang in LANGS: put("chars_request." + lang, grp(tot[lang]["chars_request"]), ["totals", lang, "chars_request"]) put("chars_reply." + lang, grp(tot[lang]["chars_reply"]), ["totals", lang, "chars_reply"]) put("chars_total." + lang, grp(tot[lang]["chars_total"]), ["totals", lang, "chars_total"]) for t in tot[lang]["tokens"]: for part in ("request", "reply", "total"): put("tokens_%s.%s.%s" % (part, t, lang), grp(tot[lang]["tokens"][t][part]), ["totals", lang, "tokens", t, part]) put("pooled_cpt.%s.%s" % (t, lang), "%.3f" % tot[lang]["pooled_chars_per_token"][t], ["totals", lang, "pooled_chars_per_token", t]) if lang != "en": put("chars_ratio." + lang, ratio(tot[lang]["chars_ratio_vs_en"]), ["totals", lang, "chars_ratio_vs_en"]) for t in tot[lang]["tokens"]: put("tokens_ratio.%s.%s" % (t, lang), ratio(tot[lang]["tokens_ratio_vs_en"][t]), ["totals", lang, "tokens_ratio_vs_en", t]) put("r_in.%s.%s" % (t, lang), ratio(tot[lang]["tokens_ratio_vs_en_request"][t]), ["totals", lang, "tokens_ratio_vs_en_request", t]) put("r_out.%s.%s" % (t, lang), ratio(tot[lang]["tokens_ratio_vs_en_reply"][t]), ["totals", lang, "tokens_ratio_vs_en_reply", t]) for m in results["models"]: if m["per_lang"] is None: continue short = m["id"].replace("gemini/", "") for lang in LANGS: pl = m["per_lang"][lang] put("cost_1000.%s.%s" % (short, lang), money(pl["cost_1000_usd"]), ["models", m["id"], "per_lang", lang, "cost_1000_usd"]) put("out_share.%s.%s" % (short, lang), pct(pl["out_share_exact"]), ["models", m["id"], "per_lang", lang, "out_share_exact"]) if lang != "en": put("ratio.%s.%s" % (short, lang), ratio(pl["ratio_vs_en"]), ["models", m["id"], "per_lang", lang, "ratio_vs_en"]) s = results["summary"] for g, byl in s["cost_ratio_range"].items(): for lang, e in byl.items(): put("cost_ratio_min.%s.%s" % (g, lang), ratio(e["min"]), ["summary", "cost_ratio_range", g, lang, "min"]) put("cost_ratio_max.%s.%s" % (g, lang), ratio(e["max"]), ["summary", "cost_ratio_range", g, lang, "max"]) put("cost_ratio_min4.%s.%s" % (g, lang), ratio4(e["min4"]), ["summary", "cost_ratio_range", g, lang, "min4"]) put("cost_ratio_max4.%s.%s" % (g, lang), ratio4(e["max4"]), ["summary", "cost_ratio_range", g, lang, "max4"]) for lang in LANGS: put("cheapest_1000." + lang, money(s["cheapest_1000"][lang]["cost"]), ["summary", "cheapest_1000", lang, "cost"]) put("priciest_1000." + lang, money(s["priciest_1000"][lang]["cost"]), ["summary", "priciest_1000", lang, "cost"]) put("out_share_min." + lang, pct(s["out_share_range"][lang]["min"]["share_exact"]), ["summary", "out_share_range", lang, "min", "share_exact"]) put("out_share_max." + lang, pct(s["out_share_range"][lang]["max"]["share_exact"]), ["summary", "out_share_range", lang, "max", "share_exact"]) for m in results["models"]: if m["per_lang"] is not None: put("out_over_in." + m["id"].replace("gemini/", ""), mult(m["out_over_in"]), ["models", m["id"], "out_over_in"]) for pp in results["per_pair"]: for g, e in s["per_pair_priced_range"].items(): for lang in ("zh_tw", "zh_cn"): put("pair_ratio.%s.%s.%s" % (pp["id"], g, lang), ratio(pp["priced_ratio_vs_en"][e["row_id"]][lang]), ["per_pair", pp["id"], "priced_ratio_vs_en", e["row_id"], lang]) for g, e in s["per_pair_priced_range"].items(): for lang in ("zh_tw", "zh_cn"): put("pair_ratio_min.%s.%s" % (g, lang), ratio(e[lang]["min"]), ["summary", "per_pair_priced_range", g, lang, "min"]) put("pair_ratio_max.%s.%s" % (g, lang), ratio(e[lang]["max"]), ["summary", "per_pair_priced_range", g, lang, "max"]) for g, byl in s["crosscheck_vs_g1"].items(): if g == "g1_source": continue for lang, e in byl.items(): put("g1_cpt.%s.%s" % (g, lang), "%.3f" % e["g1"], ["summary", "crosscheck_vs_g1", g, lang, "g1"]) ex = s["example_mix"] put("example.out_share", pct(ex["out_share_of_english_bill"]), ["summary", "example_mix", "out_share_of_english_bill"]) put("example.in_share", pct(ex["in_share_of_english_bill"]), ["summary", "example_mix", "in_share_of_english_bill"]) put("example.bill_en", usd0(ex["english_bill_usd"]), ["summary", "example_mix", "english_bill_usd"]) for g in ("o200k", "gemini"): if g in ex: for lang in ("zh_tw", "zh_cn"): put("example.ratio.%s.%s" % (g, lang), ratio(ex[g][lang]["ratio"]), ["summary", "example_mix", g, lang, "ratio"]) put("example.bill.%s.%s" % (g, lang), usd0(ex[g][lang]["bill_usd"]), ["summary", "example_mix", g, lang, "bill_usd"]) put("reply_to_request_tokens_en", ratio(s["reply_to_request_tokens_en_o200k"]), ["summary", "reply_to_request_tokens_en_o200k"]) put("out_share_if_equal_length.min", pct(s["out_share_if_equal_length"]["min"]), ["summary", "out_share_if_equal_length", "min"]) put("out_share_if_equal_length.max", pct(s["out_share_if_equal_length"]["max"]), ["summary", "out_share_if_equal_length", "max"]) return disp, src # -------------------------------------------------------------------------- # checks() # D = discriminating (fails when the thing it names goes wrong) # S = structural / same-path (cannot catch a wrong-but-consistent result) # -------------------------------------------------------------------------- def expected_tokenizer(model_id, vendor, ok_ids): """Independent of basis_tokenizer_for(): derive the tokenizer from id/vendor.""" if model_id.startswith("gemini/"): gid = model_id[len("gemini/"):] return gid if gid in ok_ids else None if model_id.startswith("gpt-"): return "o200k_base" return None def run_checks(results, pricing, corpus): """Return a list of (name, kind, ok, detail).""" out = [] def rec(name, kind, ok, detail=""): out.append((name, kind, bool(ok), detail)) ok_ids = results["gemini_models_measured"] tot = results["totals"] pairs = {p["id"]: p for p in results["corpus"]["pairs"]} # 1 corpus checks (re-run on the corpus the results were built from) cc = corpus_checks(corpus) rec("1_corpus_checks", "S", not cc, "; ".join(cc[:3])) # 2 pricing shape rows = pricing["chat"] shape_ok = len(rows) == 13 and all(r["in"] > 0 and r["out"] > 0 for r in rows) claude = [m for m in results["models"] if m["vendor"] == "Anthropic"] shape_ok = shape_ok and len(claude) == 4 and all( m["basis"] == "n/a" and m["per_lang"] is None and m["status"] == "not measured" for m in claude) shape_ok = shape_ok and results["claude_measured"] is False rec("2_pricing_shape_claude_not_measured", "S", shape_ok) # 3 cross-field binding (models vs totals, tokenizer from an independent rule) bad = [] for m in results["models"]: exp = expected_tokenizer(m["id"], m["vendor"], ok_ids) if m["vendor"] == "Anthropic": continue if m["basis_tokenizer"] != exp: bad.append("%s basis_tokenizer %s != expected %s" % (m["id"], m["basis_tokenizer"], exp)) continue if exp is None: if m["per_lang"] is not None or m["basis"] != "n/a": bad.append("%s should be n/a" % m["id"]) continue for lang in LANGS: pl = m["per_lang"][lang] if pl["tokens_in"] != tot[lang]["tokens"][exp]["request"] or pl["tokens_out"] != tot[lang]["tokens"][exp]["reply"]: bad.append("%s %s tokens do not match totals[%s]" % (m["id"], lang, exp)) rec("3_models_bound_to_right_tokenizer_columns", "D", not bad, "; ".join(bad[:3])) # 4 live re-encode of all 60 segments with tiktoken; text and chars vs corpus enc = {n: tiktoken.get_encoding(n) for n in GPT_ENCODINGS} bad = [] live_tot = {l: {n: {"request": 0, "reply": 0} for n in GPT_ENCODINGS} for l in LANGS} must = {("p01", "en", "request"), ("p05", "zh_tw", "reply"), ("p09", "zh_cn", "reply")} seen_must = set() for pid, lang, role, text in iter_segments(corpus): seg = pairs[pid]["per_lang"][lang][role] if seg["text"] != text or seg["chars"] != len(text): bad.append("%s %s %s text/chars" % (pid, lang, role)) for n in GPT_ENCODINGS: live = len(enc[n].encode(text)) live_tot[lang][n][role] += live if seg["tokens"][n] != live: bad.append("%s %s %s %s stored %d live %d" % (pid, lang, role, n, seg["tokens"][n], live)) if (pid, lang, role) in must: seen_must.add((pid, lang, role)) rec("4_live_reencode_60_segments", "D", not bad and seen_must == must, "; ".join(bad[:3])) bad = [] for lang in LANGS: for n in GPT_ENCODINGS: for role in ROLES: if tot[lang]["tokens"][n][role] != live_tot[lang][n][role]: bad.append("totals %s %s %s" % (lang, n, role)) rec("4b_totals_equal_live_sum", "D", not bad, "; ".join(bad[:3])) # 5 gemini net = raw - baseline, baseline >= 0, totals equal sum of pairs, failed ids absent bad = [] for g in GEMINI_MODELS: st = results["gemini_model_status"][g] if st["status"] != "ok": if g in ok_ids: bad.append("%s failed but listed measured" % g) for m in results["models"]: if m["id"] == "gemini/" + g and (m["basis"] != "n/a" or m["per_lang"] is not None): bad.append("%s model row should be n/a" % g) continue if g not in ok_ids or st["baseline_tokens_empty_string"] < 0: bad.append("%s ok but baseline/measured list wrong" % g) continue base = st["baseline_tokens_empty_string"] for lang in LANGS: net_sum = {"request": 0, "reply": 0} raw_sum = 0 for p in pairs.values(): for role in ROLES: seg = p["per_lang"][lang][role] if seg["tokens"][g] != seg["tokens_raw_with_overhead"][g] - base: bad.append("%s net != raw-baseline" % g) net_sum[role] += seg["tokens"][g] raw_sum += seg["tokens_raw_with_overhead"][g] if net_sum["request"] != tot[lang]["tokens"][g]["request"] or net_sum["reply"] != tot[lang]["tokens"][g]["reply"] \ or raw_sum != tot[lang]["tokens"][g]["raw_with_overhead"]: bad.append("%s %s totals != sum of pairs" % (g, lang)) rec("5_gemini_net_raw_baseline_failed_ids", "D", not bad, "; ".join(bad[:3])) # 6 cost recompute from the stored token integers and the pricing file (same formula path) bad = [] price = {r["id"]: r for r in rows} for m in results["models"]: if m["per_lang"] is None: continue r = price[m["id"]] for lang in LANGS: pl = m["per_lang"][lang] want = (pl["tokens_in"] * r["in"] + pl["tokens_out"] * r["out"]) / 1e6 * 100 if abs(pl["cost_1000_usd"] - want) > 5e-5 + 1e-9: bad.append("%s %s cost" % (m["id"], lang)) rec("6_cost_recompute", "S", not bad, "; ".join(bad[:3])) # 6b squeeze: ratio lies between r_in and r_out of the row's own tokenizer bad = [] for m in results["models"]: if m["per_lang"] is None: continue tok = expected_tokenizer(m["id"], m["vendor"], ok_ids) for lang in ("zh_tw", "zh_cn"): r_in = tot[lang]["tokens"][tok]["request"] / tot["en"]["tokens"][tok]["request"] r_out = tot[lang]["tokens"][tok]["reply"] / tot["en"]["tokens"][tok]["reply"] v = m["per_lang"][lang]["ratio_vs_en"] if not (min(r_in, r_out) - 5e-4 <= v <= max(r_in, r_out) + 5e-4): bad.append("%s %s ratio %.3f outside [%.3f, %.3f]" % (m["id"], lang, v, min(r_in, r_out), max(r_in, r_out))) # stored ratio must also equal the ratio of the stored costs want = m["per_lang"][lang]["cost_1000_usd"] / m["per_lang"]["en"]["cost_1000_usd"] if abs(v - want) > 2e-3: bad.append("%s %s ratio vs cost" % (m["id"], lang)) rec("6b_ratio_between_r_in_and_r_out", "D", not bad, "; ".join(bad[:3])) # 7 structural bad = [] for p in pairs.values(): for lang in LANGS: for role in ROLES: seg = p["per_lang"][lang][role] if seg["chars"] < 1 or any(v < 1 for v in seg["tokens"].values()): bad.append("%s %s %s zero" % (p["id"], lang, role)) for m in results["models"]: if m["per_lang"] is not None and m["per_lang"]["en"]["ratio_vs_en"] != 1.0: bad.append("%s en ratio != 1.0" % m["id"]) rec("7_structural_nonzero_en_ratio_1", "S", not bad, "; ".join(bad[:3])) # 8 display vs display_sources bad = [] disp, src = results["display"], results["display_sources"] for key, text in disp.items(): if key not in src: bad.append("%s has no source" % key) continue try: shown = float(re.sub(r"[$,×%\s]", "", text)) want = float(resolve(results, src[key])) except Exception as exc: # unresolved path or unparsable text bad.append("%s: %s" % (key, type(exc).__name__)) continue if text.endswith("%"): want, tol = want * 100, 0.5 elif text.startswith("$"): tol = 5e-5 elif "×" in text: tol = 5e-4 elif "." in text: tol = 5e-4 else: tol = 0.5 if abs(shown - want) > tol + 1e-9: bad.append("%s shows %s but source is %s" % (key, text, want)) for key in src: if key not in disp: bad.append("%s source without display" % key) rec("8_display_equals_sources", "D", not bad, "; ".join(bad[:3])) # 9 per-exchange priced ratios and the worked example, recomputed from the segment token # integers and the pricing file (not from the stored per_pair / totals / models figures) bad = [] if [x["id"] for x in results["per_pair"]] != [p["id"] for p in results["corpus"]["pairs"]]: bad.append("per_pair ids differ from corpus pair ids") stored = {x["id"]: x for x in results["per_pair"]} sum_cost = {} # (model id, lang) -> USD for all pairs, from segment tokens for pid, pr in pairs.items(): for r in rows: if r["vendor"] == "Anthropic": continue tok = expected_tokenizer(r["id"], r["vendor"], ok_ids) c = {} for lang in LANGS: tin = pr["per_lang"][lang]["request"]["tokens"][tok] tout = pr["per_lang"][lang]["reply"]["tokens"][tok] c[lang] = (tin * r["in"] + tout * r["out"]) / 1e6 sum_cost[(r["id"], lang)] = sum_cost.get((r["id"], lang), 0.0) + c[lang] for lang in ("zh_tw", "zh_cn"): want = c[lang] / c["en"] got = stored[pid]["priced_ratio_vs_en"][r["id"]][lang] if abs(got - want) > 5e-4 + 1e-9: bad.append("%s %s %s priced ratio %.3f vs %.4f" % (pid, r["id"], lang, got, want)) for m in results["models"]: if m["per_lang"] is None: continue for lang in LANGS: if abs(sum_cost[(m["id"], lang)] * 100 - m["per_lang"][lang]["cost_1000_usd"]) > 5e-5 + 1e-9: bad.append("%s %s per-pair costs do not add up to the model row" % (m["id"], lang)) ex = results["summary"]["example_mix"] for g in ("o200k", "gemini"): if g not in ex: continue tok = ex[g]["basis_tokenizer"] if (g == "o200k" and tok != "o200k_base") or (g == "gemini" and tok not in ok_ids): bad.append("example %s uses tokenizer %s" % (g, tok)) continue for lang in ("zh_tw", "zh_cn"): r_in = tot[lang]["tokens"][tok]["request"] / tot["en"]["tokens"][tok]["request"] r_out = tot[lang]["tokens"][tok]["reply"] / tot["en"]["tokens"][tok]["reply"] want = ex["in_share_of_english_bill"] * r_in + ex["out_share_of_english_bill"] * r_out if abs(ex[g][lang]["ratio"] - want) > 5e-4 + 1e-9 or not min(r_in, r_out) - 5e-4 <= ex[g][lang]["ratio"] <= max(r_in, r_out) + 5e-4: bad.append("example %s %s ratio %.3f vs %.4f" % (g, lang, ex[g][lang]["ratio"], want)) if abs(ex[g][lang]["bill_usd"] - ex["english_bill_usd"] * want) > 0.5 + 1e-6: bad.append("example %s %s bill" % (g, lang)) for g, e in results["summary"]["per_pair_priced_range"].items(): for lang in ("zh_tw", "zh_cn"): vals = [x["priced_ratio_vs_en"][e["row_id"]][lang] for x in results["per_pair"]] if abs(e[lang]["min"] - min(vals)) > 1e-9 or abs(e[lang]["max"] - max(vals)) > 1e-9: bad.append("per_pair_priced_range %s %s" % (g, lang)) rec("9_per_pair_and_example_recompute", "D", not bad, "; ".join(bad[:3])) return out def print_checks(checks): for name, kind, ok, detail in checks: print(" [%s] %-44s %s %s" % (kind, name, "PASS" if ok else "FAIL", detail)) # -------------------------------------------------------------------------- # main # -------------------------------------------------------------------------- def corpus_table(corpus): print("%-4s %-46s %9s %9s %9s %9s %9s %9s" % ("id", "topic", "en_req", "en_rep", "tw_req", "tw_rep", "cn_req", "cn_rep")) for p in corpus: print("%-4s %-46s %9d %9d %9d %9d %9d %9d" % ( p["id"], p["topic"], len(p["en"]["request"]), len(p["en"]["reply"]), len(p["zh_tw"]["request"]), len(p["zh_tw"]["reply"]), len(p["zh_cn"]["request"]), len(p["zh_cn"]["reply"]))) def main(argv=None): if hasattr(sys.stdout, "reconfigure"): sys.stdout.reconfigure(encoding="utf-8") ap = argparse.ArgumentParser() ap.add_argument("--offline", action="store_true") ap.add_argument("--check-corpus", action="store_true") ap.add_argument("--pricing", default=str(DEFAULT_PRICING)) ap.add_argument("--out", default=str(DEFAULT_OUT)) args = ap.parse_args(argv) corpus = CORPUS bad = corpus_checks(corpus) if bad: print("corpus_checks FAILED:") for b in bad: print(" -", b) raise SystemExit(1) print("corpus_checks passed (%d exchanges, %d passages)" % (len(corpus), len(corpus) * 6)) if args.check_corpus: corpus_table(corpus) return 0 pricing = json.loads(Path(args.pricing).read_text(encoding="utf-8")) if args.offline: counts = load_counts_offline(args.out, corpus) else: counts = measure_online(corpus) counts["gpt_tokens"] = gpt_counts(corpus) results = build_results(counts, pricing, corpus) checks = run_checks(results, pricing, corpus) results["checks"] = {name: ok for name, kind, ok, detail in checks} results["checks_kind"] = {name: kind for name, kind, ok, detail in checks} print("checks (D = discriminating, S = structural / same-path):") print_checks(checks) if not all(ok for _, _, ok, _ in checks): print("CHECKS FAILED; results file not written.") raise SystemExit(1) Path(args.out).write_text(json.dumps(results, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", newline="\n") print("Wrote", Path(args.out).name) print("gemini models measured:", ", ".join(results["gemini_models_measured"]) or "none") for lang in LANGS: t = results["totals"][lang] print("%-6s chars=%5d o200k=%5d cl100k=%5d" % ( lang, t["chars_total"], t["tokens"]["o200k_base"]["total"], t["tokens"]["cl100k_base"]["total"])) return 0 if __name__ == "__main__": sys.exit(main())