safe_v2 flexitok/supertokenizer-safe_v2 Updated May 2 • 1 flexitok/bpe_arb_Arab_8000 Updated May 2 flexitok/bpe_ces_Latn_8000 Updated May 2 flexitok/bpe_ltr_cmn_Hani_8000_v2 Updated May 2
safe flexitok/supertokenizer-safe Updated Apr 29 • 10 flexitok/bpe_arb_Arab_8000 Updated May 2 flexitok/bpe_ces_Latn_8000 Updated May 2 flexitok/bpe_ltr_cmn_Hani_8000_v2 Updated May 2
8000-vocab-v3 flexitok/bpe_ltr_tur_Latn_8000_v3 Updated Apr 29 flexitok/bpe_ltr_nld_Latn_8000_v3 Updated Apr 29 flexitok/bpe_ltr_por_Latn_8000_v3 Updated Apr 29 flexitok/bpe_ltr_jpn_Jpan_8000_v3 Updated Apr 29
maddition_360 flexitok/supertokenizer-maddition_360 Updated Apr 21 • 3 flexitok/maddition_jpn_Jpan_360 Updated May 1 flexitok/maddition_cmn_Hani_360 Updated May 1 flexitok/maddition_ita_Latn_360 Updated May 1
maddition flexitok/supertokenizer-maddition Updated Apr 18 • 6 flexitok/maddition_jpn_Jpan_2000 Updated Apr 19 flexitok/maddition_cmn_Hani_2000 Updated Apr 19 flexitok/maddition_ita_Latn_2000 Updated Apr 19
script_1 flexitok/supertokenizer-script_1 Updated Apr 18 • 4 flexitok/bpe_script_Arab_16000 Updated Apr 18 flexitok/bpe_script_CmJp_16000 Updated Apr 18 flexitok/bpe_ltr_ell_Grek_8000_v2 Updated Apr 18
llama_and_16k flexitok/supertokenizer-llama_and_16k Updated Apr 16 • 5 flexitok/bpe_ltr_arb_Arab_16000_v2 Updated Apr 16 flexitok/bpe_ltr_ces_Latn_16000_v2 Updated Apr 16 flexitok/bpe_ltr_cmn_Hani_16000_v2 Updated Apr 16
8k_v2 flexitok/supertokenizer-8k_v2 Updated Apr 16 • 9 flexitok/bpe_ltr_arb_Arab_8000_v2 Updated Apr 16 flexitok/bpe_ltr_ces_Latn_8000_v2 Updated Apr 16 flexitok/bpe_ltr_cmn_Hani_8000_v2 Updated May 2
16000-vocab-v2 flexitok/bpe_tur_Latn_16000_v2 Updated Apr 14 flexitok/bpe_ita_Latn_16000_v2 Updated Apr 14 flexitok/bpe_fas_Arab_16000_v2 Updated Apr 14 flexitok/bpe_arb_Arab_16000_v2 Updated Apr 14
1000-vocab-v2 flexitok/bpe_tur_Latn_1000_v2 Updated Apr 13 flexitok/bpe_ltr_vie_Latn_1000_v2 Updated Apr 15 flexitok/bpe_ltr_tur_Latn_1000_v2 Updated Apr 15 flexitok/bpe_ltr_dan_Latn_1000_v2 Updated Apr 15
fineweb2-hq-tokenizers-v2 flexitok/bpe_vie_Latn_4000_v2 Updated Apr 13 flexitok/bpe_dan_Latn_4000_v2 Updated Apr 13 flexitok/bpe_swe_Latn_4000_v2 Updated Apr 13 flexitok/bpe_fas_Arab_4000_v2 Updated Apr 13
flexitok-mod-tokenizers-zero-padded flexitok/supertokenizer-mod_tokenizers_zero_padded Updated Mar 3 • 11 flexitok/mod-tokenizers-zero-padded-individual Updated Mar 3 flexitok/mod-tokenizers-zero-padded-ltr_3digit 3.97M • Updated Mar 4 • 1 flexitok/mod-tokenizers-zero-padded-ltr_2digit 3.46M • Updated Mar 4 • 10
flexitok/fineweb2-hq-superset-tokenizers flexitok/bpe_arb_Arab_8000 Updated May 2 flexitok/bpe_ces_Latn_8000 Updated May 2 flexitok/bpe_cmn_Hani_8000 Updated Feb 25 flexitok/bpe_dan_Latn_8000 Updated May 2
merging-toksuite moe-dtoks/qwen_onto_llama_lambda-0 Updated Feb 16 • 6 moe-dtoks/llama_onto_qwen_lambda-0 Updated Feb 16 • 3 moe-dtoks/llama_onto_qwen_lambda-1-nse-random Updated Feb 16 • 3 moe-dtoks/llama_onto_qwen_lambda-1 Updated Feb 16 • 2
maddition_360_v2 flexitok/supertokenizer-maddition_360_v2 Updated May 1 • 6 flexitok/maddition_jpn_Jpan_360 Updated May 1 flexitok/maddition_cmn_Hani_360 Updated May 1 flexitok/maddition_ita_Latn_360 Updated May 1
8k_v3 flexitok/supertokenizer-8k_v3 Updated Apr 29 • 2 flexitok/bpe_ltr_arb_Arab_8000_v3 Updated Apr 29 flexitok/bpe_ltr_ces_Latn_8000_v3 Updated Apr 29 flexitok/bpe_ltr_cmn_Hani_8000_v3 Updated Apr 29
fineweb2-hq-tokenizers-v3 flexitok/bpe_ltr_tur_Latn_8000_v3 Updated Apr 29 flexitok/bpe_ltr_nld_Latn_8000_v3 Updated Apr 29 flexitok/bpe_ltr_por_Latn_8000_v3 Updated Apr 29 flexitok/bpe_ltr_jpn_Jpan_8000_v3 Updated Apr 29
multilingual_addition_tokenizers_minimal flexitok/maddition_AllL_3_000_minimal Updated Apr 21 flexitok/maddition_digit_v2_360 Updated May 1 flexitok/maddition_AllL_2000_minimal Updated May 7
multilingual_addition_tokenizers flexitok/maddition_jpn_Jpan_2000 Updated Apr 19 flexitok/maddition_cmn_Hani_2000 Updated Apr 19 flexitok/maddition_ita_Latn_2000 Updated Apr 19 flexitok/maddition_por_Latn_2000 Updated Apr 19
script-based-tokenizers flexitok/bpe_boundless_AllL_128000 Updated Apr 29 flexitok/bpe_boundless_coverage_AllL_128000 Updated Apr 29 flexitok/bpe_gpt4_AllL_128000 Updated Apr 29 flexitok/bpe_gpt4_AllL_323000 Updated Apr 29
16k_v2 flexitok/supertokenizer-16k_v2 Updated Apr 16 • 10 flexitok/bpe_ltr_arb_Arab_16000_v2 Updated Apr 16 flexitok/bpe_ltr_ces_Latn_16000_v2 Updated Apr 16 flexitok/bpe_ltr_cmn_Hani_16000_v2 Updated Apr 16
32000-vocab-v2 flexitok/bpe_hun_Latn_32000_v2 Updated Apr 14 flexitok/bpe_tur_Latn_32000_v2 Updated Apr 14 flexitok/bpe_dan_Latn_32000_v2 Updated Apr 14 flexitok/bpe_swe_Latn_32000_v2 Updated Apr 14
8000-vocab-v2 flexitok/bpe_tur_Latn_8000_v2 Updated Apr 14 flexitok/bpe_ind_Latn_8000_v2 Updated Apr 14 flexitok/bpe_dan_Latn_8000_v2 Updated Apr 14 flexitok/bpe_vie_Latn_8000_v2 Updated Apr 14
4000-vocab-v2 flexitok/bpe_vie_Latn_4000_v2 Updated Apr 13 flexitok/bpe_dan_Latn_4000_v2 Updated Apr 13 flexitok/bpe_swe_Latn_4000_v2 Updated Apr 13 flexitok/bpe_fas_Arab_4000_v2 Updated Apr 13
4000-vocab flexitok/bpe_vie_Latn_4000 Updated Apr 11 flexitok/bpe_swe_Latn_4000 Updated Apr 11 flexitok/bpe_dan_Latn_4000 Updated Apr 11 flexitok/bpe_hun_Latn_4000 Updated Apr 11
flexitok/mod-tokenizers flexitok/mod-tokenizers-individual 3.42M • Updated Mar 3 • 8 flexitok/mod-tokenizers-ltr_3digit 3.93M • Updated Mar 3 • 10 flexitok/mod-tokenizers-ltr_2digit 3.46M • Updated Mar 3 • 6 flexitok/mod-tokenizers-ltr_4digit 8.53M • Updated Mar 3 • 6
Fineweb2-HQ-Tokenizers flexitok/unigram_fw_edu_16000 Updated Feb 23 flexitok/bpe_vie_Latn_16000 Updated Apr 10 flexitok/bpe_ita_Latn_16000 Updated Apr 10 flexitok/bpe_rus_Cyrl_16000 Updated Apr 10
baselines flexitok/llama_baseline_english 2B • Updated Jan 23 • 9 flexitok/llama_bpe_dropout_english 2B • Updated Jan 23 • 4 flexitok/llama_baseline 2B • Updated Apr 8 • 4 flexitok/llama_bpe_dropout 2B • Updated Apr 9 • 31
safe_v2 flexitok/supertokenizer-safe_v2 Updated May 2 • 1 flexitok/bpe_arb_Arab_8000 Updated May 2 flexitok/bpe_ces_Latn_8000 Updated May 2 flexitok/bpe_ltr_cmn_Hani_8000_v2 Updated May 2
maddition_360_v2 flexitok/supertokenizer-maddition_360_v2 Updated May 1 • 6 flexitok/maddition_jpn_Jpan_360 Updated May 1 flexitok/maddition_cmn_Hani_360 Updated May 1 flexitok/maddition_ita_Latn_360 Updated May 1
safe flexitok/supertokenizer-safe Updated Apr 29 • 10 flexitok/bpe_arb_Arab_8000 Updated May 2 flexitok/bpe_ces_Latn_8000 Updated May 2 flexitok/bpe_ltr_cmn_Hani_8000_v2 Updated May 2
8k_v3 flexitok/supertokenizer-8k_v3 Updated Apr 29 • 2 flexitok/bpe_ltr_arb_Arab_8000_v3 Updated Apr 29 flexitok/bpe_ltr_ces_Latn_8000_v3 Updated Apr 29 flexitok/bpe_ltr_cmn_Hani_8000_v3 Updated Apr 29
8000-vocab-v3 flexitok/bpe_ltr_tur_Latn_8000_v3 Updated Apr 29 flexitok/bpe_ltr_nld_Latn_8000_v3 Updated Apr 29 flexitok/bpe_ltr_por_Latn_8000_v3 Updated Apr 29 flexitok/bpe_ltr_jpn_Jpan_8000_v3 Updated Apr 29
fineweb2-hq-tokenizers-v3 flexitok/bpe_ltr_tur_Latn_8000_v3 Updated Apr 29 flexitok/bpe_ltr_nld_Latn_8000_v3 Updated Apr 29 flexitok/bpe_ltr_por_Latn_8000_v3 Updated Apr 29 flexitok/bpe_ltr_jpn_Jpan_8000_v3 Updated Apr 29
maddition_360 flexitok/supertokenizer-maddition_360 Updated Apr 21 • 3 flexitok/maddition_jpn_Jpan_360 Updated May 1 flexitok/maddition_cmn_Hani_360 Updated May 1 flexitok/maddition_ita_Latn_360 Updated May 1
multilingual_addition_tokenizers_minimal flexitok/maddition_AllL_3_000_minimal Updated Apr 21 flexitok/maddition_digit_v2_360 Updated May 1 flexitok/maddition_AllL_2000_minimal Updated May 7
maddition flexitok/supertokenizer-maddition Updated Apr 18 • 6 flexitok/maddition_jpn_Jpan_2000 Updated Apr 19 flexitok/maddition_cmn_Hani_2000 Updated Apr 19 flexitok/maddition_ita_Latn_2000 Updated Apr 19
multilingual_addition_tokenizers flexitok/maddition_jpn_Jpan_2000 Updated Apr 19 flexitok/maddition_cmn_Hani_2000 Updated Apr 19 flexitok/maddition_ita_Latn_2000 Updated Apr 19 flexitok/maddition_por_Latn_2000 Updated Apr 19
script_1 flexitok/supertokenizer-script_1 Updated Apr 18 • 4 flexitok/bpe_script_Arab_16000 Updated Apr 18 flexitok/bpe_script_CmJp_16000 Updated Apr 18 flexitok/bpe_ltr_ell_Grek_8000_v2 Updated Apr 18
script-based-tokenizers flexitok/bpe_boundless_AllL_128000 Updated Apr 29 flexitok/bpe_boundless_coverage_AllL_128000 Updated Apr 29 flexitok/bpe_gpt4_AllL_128000 Updated Apr 29 flexitok/bpe_gpt4_AllL_323000 Updated Apr 29
llama_and_16k flexitok/supertokenizer-llama_and_16k Updated Apr 16 • 5 flexitok/bpe_ltr_arb_Arab_16000_v2 Updated Apr 16 flexitok/bpe_ltr_ces_Latn_16000_v2 Updated Apr 16 flexitok/bpe_ltr_cmn_Hani_16000_v2 Updated Apr 16
16k_v2 flexitok/supertokenizer-16k_v2 Updated Apr 16 • 10 flexitok/bpe_ltr_arb_Arab_16000_v2 Updated Apr 16 flexitok/bpe_ltr_ces_Latn_16000_v2 Updated Apr 16 flexitok/bpe_ltr_cmn_Hani_16000_v2 Updated Apr 16
8k_v2 flexitok/supertokenizer-8k_v2 Updated Apr 16 • 9 flexitok/bpe_ltr_arb_Arab_8000_v2 Updated Apr 16 flexitok/bpe_ltr_ces_Latn_8000_v2 Updated Apr 16 flexitok/bpe_ltr_cmn_Hani_8000_v2 Updated May 2
32000-vocab-v2 flexitok/bpe_hun_Latn_32000_v2 Updated Apr 14 flexitok/bpe_tur_Latn_32000_v2 Updated Apr 14 flexitok/bpe_dan_Latn_32000_v2 Updated Apr 14 flexitok/bpe_swe_Latn_32000_v2 Updated Apr 14
16000-vocab-v2 flexitok/bpe_tur_Latn_16000_v2 Updated Apr 14 flexitok/bpe_ita_Latn_16000_v2 Updated Apr 14 flexitok/bpe_fas_Arab_16000_v2 Updated Apr 14 flexitok/bpe_arb_Arab_16000_v2 Updated Apr 14
8000-vocab-v2 flexitok/bpe_tur_Latn_8000_v2 Updated Apr 14 flexitok/bpe_ind_Latn_8000_v2 Updated Apr 14 flexitok/bpe_dan_Latn_8000_v2 Updated Apr 14 flexitok/bpe_vie_Latn_8000_v2 Updated Apr 14
1000-vocab-v2 flexitok/bpe_tur_Latn_1000_v2 Updated Apr 13 flexitok/bpe_ltr_vie_Latn_1000_v2 Updated Apr 15 flexitok/bpe_ltr_tur_Latn_1000_v2 Updated Apr 15 flexitok/bpe_ltr_dan_Latn_1000_v2 Updated Apr 15
4000-vocab-v2 flexitok/bpe_vie_Latn_4000_v2 Updated Apr 13 flexitok/bpe_dan_Latn_4000_v2 Updated Apr 13 flexitok/bpe_swe_Latn_4000_v2 Updated Apr 13 flexitok/bpe_fas_Arab_4000_v2 Updated Apr 13
fineweb2-hq-tokenizers-v2 flexitok/bpe_vie_Latn_4000_v2 Updated Apr 13 flexitok/bpe_dan_Latn_4000_v2 Updated Apr 13 flexitok/bpe_swe_Latn_4000_v2 Updated Apr 13 flexitok/bpe_fas_Arab_4000_v2 Updated Apr 13
4000-vocab flexitok/bpe_vie_Latn_4000 Updated Apr 11 flexitok/bpe_swe_Latn_4000 Updated Apr 11 flexitok/bpe_dan_Latn_4000 Updated Apr 11 flexitok/bpe_hun_Latn_4000 Updated Apr 11
flexitok-mod-tokenizers-zero-padded flexitok/supertokenizer-mod_tokenizers_zero_padded Updated Mar 3 • 11 flexitok/mod-tokenizers-zero-padded-individual Updated Mar 3 flexitok/mod-tokenizers-zero-padded-ltr_3digit 3.97M • Updated Mar 4 • 1 flexitok/mod-tokenizers-zero-padded-ltr_2digit 3.46M • Updated Mar 4 • 10
flexitok/mod-tokenizers flexitok/mod-tokenizers-individual 3.42M • Updated Mar 3 • 8 flexitok/mod-tokenizers-ltr_3digit 3.93M • Updated Mar 3 • 10 flexitok/mod-tokenizers-ltr_2digit 3.46M • Updated Mar 3 • 6 flexitok/mod-tokenizers-ltr_4digit 8.53M • Updated Mar 3 • 6
flexitok/fineweb2-hq-superset-tokenizers flexitok/bpe_arb_Arab_8000 Updated May 2 flexitok/bpe_ces_Latn_8000 Updated May 2 flexitok/bpe_cmn_Hani_8000 Updated Feb 25 flexitok/bpe_dan_Latn_8000 Updated May 2
Fineweb2-HQ-Tokenizers flexitok/unigram_fw_edu_16000 Updated Feb 23 flexitok/bpe_vie_Latn_16000 Updated Apr 10 flexitok/bpe_ita_Latn_16000 Updated Apr 10 flexitok/bpe_rus_Cyrl_16000 Updated Apr 10
merging-toksuite moe-dtoks/qwen_onto_llama_lambda-0 Updated Feb 16 • 6 moe-dtoks/llama_onto_qwen_lambda-0 Updated Feb 16 • 3 moe-dtoks/llama_onto_qwen_lambda-1-nse-random Updated Feb 16 • 3 moe-dtoks/llama_onto_qwen_lambda-1 Updated Feb 16 • 2
baselines flexitok/llama_baseline_english 2B • Updated Jan 23 • 9 flexitok/llama_bpe_dropout_english 2B • Updated Jan 23 • 4 flexitok/llama_baseline 2B • Updated Apr 8 • 4 flexitok/llama_bpe_dropout 2B • Updated Apr 9 • 31
