{
  "encodings_measured": [
    "cl100k_base",
    "o200k_base"
  ],
  "categories": {
    "english_prose": {
      "sample_count": 10,
      "by_encoding": {
        "cl100k_base": {
          "avg_chars_per_token": 5.153,
          "min_chars_per_token": 4.588,
          "max_chars_per_token": 5.643,
          "samples": [
            {
              "chars": 168,
              "tokens": 33,
              "chars_per_token": 5.091
            },
            {
              "chars": 192,
              "tokens": 37,
              "chars_per_token": 5.189
            },
            {
              "chars": 162,
              "tokens": 33,
              "chars_per_token": 4.909
            },
            {
              "chars": 195,
              "tokens": 37,
              "chars_per_token": 5.27
            },
            {
              "chars": 170,
              "tokens": 32,
              "chars_per_token": 5.312
            },
            {
              "chars": 153,
              "tokens": 29,
              "chars_per_token": 5.276
            },
            {
              "chars": 168,
              "tokens": 32,
              "chars_per_token": 5.25
            },
            {
              "chars": 158,
              "tokens": 28,
              "chars_per_token": 5.643
            },
            {
              "chars": 160,
              "tokens": 32,
              "chars_per_token": 5.0
            },
            {
              "chars": 156,
              "tokens": 34,
              "chars_per_token": 4.588
            }
          ]
        },
        "o200k_base": {
          "avg_chars_per_token": 5.233,
          "min_chars_per_token": 4.588,
          "max_chars_per_token": 5.643,
          "samples": [
            {
              "chars": 168,
              "tokens": 33,
              "chars_per_token": 5.091
            },
            {
              "chars": 192,
              "tokens": 37,
              "chars_per_token": 5.189
            },
            {
              "chars": 162,
              "tokens": 32,
              "chars_per_token": 5.062
            },
            {
              "chars": 195,
              "tokens": 35,
              "chars_per_token": 5.571
            },
            {
              "chars": 170,
              "tokens": 32,
              "chars_per_token": 5.312
            },
            {
              "chars": 153,
              "tokens": 28,
              "chars_per_token": 5.464
            },
            {
              "chars": 168,
              "tokens": 32,
              "chars_per_token": 5.25
            },
            {
              "chars": 158,
              "tokens": 28,
              "chars_per_token": 5.643
            },
            {
              "chars": 160,
              "tokens": 31,
              "chars_per_token": 5.161
            },
            {
              "chars": 156,
              "tokens": 34,
              "chars_per_token": 4.588
            }
          ]
        }
      }
    },
    "chinese_traditional": {
      "sample_count": 10,
      "by_encoding": {
        "cl100k_base": {
          "avg_chars_per_token": 0.777,
          "min_chars_per_token": 0.702,
          "max_chars_per_token": 0.868,
          "samples": [
            {
              "chars": 59,
              "tokens": 68,
              "chars_per_token": 0.868
            },
            {
              "chars": 50,
              "tokens": 62,
              "chars_per_token": 0.806
            },
            {
              "chars": 47,
              "tokens": 64,
              "chars_per_token": 0.734
            },
            {
              "chars": 43,
              "tokens": 57,
              "chars_per_token": 0.754
            },
            {
              "chars": 40,
              "tokens": 55,
              "chars_per_token": 0.727
            },
            {
              "chars": 40,
              "tokens": 57,
              "chars_per_token": 0.702
            },
            {
              "chars": 57,
              "tokens": 73,
              "chars_per_token": 0.781
            },
            {
              "chars": 51,
              "tokens": 65,
              "chars_per_token": 0.785
            },
            {
              "chars": 44,
              "tokens": 54,
              "chars_per_token": 0.815
            },
            {
              "chars": 51,
              "tokens": 64,
              "chars_per_token": 0.797
            }
          ]
        },
        "o200k_base": {
          "avg_chars_per_token": 1.178,
          "min_chars_per_token": 1.103,
          "max_chars_per_token": 1.275,
          "samples": [
            {
              "chars": 59,
              "tokens": 47,
              "chars_per_token": 1.255
            },
            {
              "chars": 50,
              "tokens": 42,
              "chars_per_token": 1.19
            },
            {
              "chars": 47,
              "tokens": 42,
              "chars_per_token": 1.119
            },
            {
              "chars": 43,
              "tokens": 39,
              "chars_per_token": 1.103
            },
            {
              "chars": 40,
              "tokens": 35,
              "chars_per_token": 1.143
            },
            {
              "chars": 40,
              "tokens": 36,
              "chars_per_token": 1.111
            },
            {
              "chars": 57,
              "tokens": 47,
              "chars_per_token": 1.213
            },
            {
              "chars": 51,
              "tokens": 40,
              "chars_per_token": 1.275
            },
            {
              "chars": 44,
              "tokens": 39,
              "chars_per_token": 1.128
            },
            {
              "chars": 51,
              "tokens": 41,
              "chars_per_token": 1.244
            }
          ]
        }
      }
    },
    "chinese_simplified": {
      "sample_count": 10,
      "by_encoding": {
        "cl100k_base": {
          "avg_chars_per_token": 0.987,
          "min_chars_per_token": 0.846,
          "max_chars_per_token": 1.136,
          "samples": [
            {
              "chars": 59,
              "tokens": 56,
              "chars_per_token": 1.054
            },
            {
              "chars": 50,
              "tokens": 49,
              "chars_per_token": 1.02
            },
            {
              "chars": 47,
              "tokens": 46,
              "chars_per_token": 1.022
            },
            {
              "chars": 43,
              "tokens": 45,
              "chars_per_token": 0.956
            },
            {
              "chars": 39,
              "tokens": 45,
              "chars_per_token": 0.867
            },
            {
              "chars": 39,
              "tokens": 43,
              "chars_per_token": 0.907
            },
            {
              "chars": 57,
              "tokens": 56,
              "chars_per_token": 1.018
            },
            {
              "chars": 51,
              "tokens": 49,
              "chars_per_token": 1.041
            },
            {
              "chars": 44,
              "tokens": 52,
              "chars_per_token": 0.846
            },
            {
              "chars": 50,
              "tokens": 44,
              "chars_per_token": 1.136
            }
          ]
        },
        "o200k_base": {
          "avg_chars_per_token": 1.401,
          "min_chars_per_token": 1.158,
          "max_chars_per_token": 1.562,
          "samples": [
            {
              "chars": 59,
              "tokens": 38,
              "chars_per_token": 1.553
            },
            {
              "chars": 50,
              "tokens": 33,
              "chars_per_token": 1.515
            },
            {
              "chars": 47,
              "tokens": 34,
              "chars_per_token": 1.382
            },
            {
              "chars": 43,
              "tokens": 32,
              "chars_per_token": 1.344
            },
            {
              "chars": 39,
              "tokens": 28,
              "chars_per_token": 1.393
            },
            {
              "chars": 39,
              "tokens": 32,
              "chars_per_token": 1.219
            },
            {
              "chars": 57,
              "tokens": 40,
              "chars_per_token": 1.425
            },
            {
              "chars": 51,
              "tokens": 35,
              "chars_per_token": 1.457
            },
            {
              "chars": 44,
              "tokens": 38,
              "chars_per_token": 1.158
            },
            {
              "chars": 50,
              "tokens": 32,
              "chars_per_token": 1.562
            }
          ]
        }
      }
    },
    "python_code": {
      "sample_count": 10,
      "by_encoding": {
        "cl100k_base": {
          "avg_chars_per_token": 3.772,
          "min_chars_per_token": 2.814,
          "max_chars_per_token": 4.293,
          "samples": [
            {
              "chars": 124,
              "tokens": 35,
              "chars_per_token": 3.543
            },
            {
              "chars": 324,
              "tokens": 87,
              "chars_per_token": 3.724
            },
            {
              "chars": 168,
              "tokens": 48,
              "chars_per_token": 3.5
            },
            {
              "chars": 217,
              "tokens": 55,
              "chars_per_token": 3.945
            },
            {
              "chars": 238,
              "tokens": 58,
              "chars_per_token": 4.103
            },
            {
              "chars": 201,
              "tokens": 52,
              "chars_per_token": 3.865
            },
            {
              "chars": 251,
              "tokens": 60,
              "chars_per_token": 4.183
            },
            {
              "chars": 176,
              "tokens": 41,
              "chars_per_token": 4.293
            },
            {
              "chars": 161,
              "tokens": 43,
              "chars_per_token": 3.744
            },
            {
              "chars": 121,
              "tokens": 43,
              "chars_per_token": 2.814
            }
          ]
        },
        "o200k_base": {
          "avg_chars_per_token": 3.742,
          "min_chars_per_token": 2.814,
          "max_chars_per_token": 4.293,
          "samples": [
            {
              "chars": 124,
              "tokens": 35,
              "chars_per_token": 3.543
            },
            {
              "chars": 324,
              "tokens": 87,
              "chars_per_token": 3.724
            },
            {
              "chars": 168,
              "tokens": 49,
              "chars_per_token": 3.429
            },
            {
              "chars": 217,
              "tokens": 55,
              "chars_per_token": 3.945
            },
            {
              "chars": 238,
              "tokens": 59,
              "chars_per_token": 4.034
            },
            {
              "chars": 201,
              "tokens": 52,
              "chars_per_token": 3.865
            },
            {
              "chars": 251,
              "tokens": 61,
              "chars_per_token": 4.115
            },
            {
              "chars": 176,
              "tokens": 41,
              "chars_per_token": 4.293
            },
            {
              "chars": 161,
              "tokens": 44,
              "chars_per_token": 3.659
            },
            {
              "chars": 121,
              "tokens": 43,
              "chars_per_token": 2.814
            }
          ]
        }
      }
    },
    "json": {
      "sample_count": 10,
      "by_encoding": {
        "cl100k_base": {
          "avg_chars_per_token": 2.658,
          "min_chars_per_token": 1.904,
          "max_chars_per_token": 3.296,
          "samples": [
            {
              "chars": 99,
              "tokens": 52,
              "chars_per_token": 1.904
            },
            {
              "chars": 80,
              "tokens": 39,
              "chars_per_token": 2.051
            },
            {
              "chars": 91,
              "tokens": 35,
              "chars_per_token": 2.6
            },
            {
              "chars": 87,
              "tokens": 27,
              "chars_per_token": 3.222
            },
            {
              "chars": 81,
              "tokens": 29,
              "chars_per_token": 2.793
            },
            {
              "chars": 81,
              "tokens": 31,
              "chars_per_token": 2.613
            },
            {
              "chars": 86,
              "tokens": 29,
              "chars_per_token": 2.966
            },
            {
              "chars": 89,
              "tokens": 27,
              "chars_per_token": 3.296
            },
            {
              "chars": 82,
              "tokens": 33,
              "chars_per_token": 2.485
            },
            {
              "chars": 90,
              "tokens": 34,
              "chars_per_token": 2.647
            }
          ]
        },
        "o200k_base": {
          "avg_chars_per_token": 2.647,
          "min_chars_per_token": 1.941,
          "max_chars_per_token": 3.296,
          "samples": [
            {
              "chars": 99,
              "tokens": 51,
              "chars_per_token": 1.941
            },
            {
              "chars": 80,
              "tokens": 39,
              "chars_per_token": 2.051
            },
            {
              "chars": 91,
              "tokens": 36,
              "chars_per_token": 2.528
            },
            {
              "chars": 87,
              "tokens": 27,
              "chars_per_token": 3.222
            },
            {
              "chars": 81,
              "tokens": 29,
              "chars_per_token": 2.793
            },
            {
              "chars": 81,
              "tokens": 31,
              "chars_per_token": 2.613
            },
            {
              "chars": 86,
              "tokens": 29,
              "chars_per_token": 2.966
            },
            {
              "chars": 89,
              "tokens": 27,
              "chars_per_token": 3.296
            },
            {
              "chars": 82,
              "tokens": 34,
              "chars_per_token": 2.412
            },
            {
              "chars": 90,
              "tokens": 34,
              "chars_per_token": 2.647
            }
          ]
        }
      }
    }
  }
}