{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2606-20993","title":"Phonemes to the Rescue: Multilingual Tokenization Based on International Phonetic Alphabet","arxiv_id":"2606.20993","date":"2026-06-18","proceeding":null,"authors":["Milan Miletić","Julie Kallini","Ekaterina Shutova"],"abstract":"Multilingual language models often exhibit performance disparities across languages that can arise as early as the tokenization stage. Widely-used subword tokenization approaches favor high-resource languages, and tokenizer-free methods still yield longer sequences for scripts with a higher bytes-per-character ratio. To address these shortcomings, we propose to use the International Phonetic Alphabet (IPA) as a language-agnostic input representation for multilingual tokenizers. IPA provides a compact symbol inventory, greater cross-lingual character overlap, and a more balanced byte-per-character distribution across languages. We train matched pairs of text vs. IPA subword tokenizers across 24 languages and 14 scripts and demonstrate that IPA tokenizers consistently improve tokenization quality, especially for non-Latin scripts, and generalize more effectively to unseen languages and scripts.","url_abs":"https://arxiv.org/abs/2606.20993","url_pdf":"https://arxiv.org/pdf/2606.20993","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2606.20993","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2606.20993"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/Mikki99/ipa-tokenization","reach":null}],"summary":{"ran":15,"unverified":1},"by_repo_kind":{"found_in_text":{"samples":16,"ran":15,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"44e55b1aa60bdfd6","entry":"apply_fix","repo":"Mikki99/ipa-tokenization","repo_kind":"found_in_text","path":"src/ipa_tok/g2p/jap_fix.py","file_url":"https://github.com/Mikki99/ipa-tokenization/blob/HEAD/src/ipa_tok/g2p/jap_fix.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"44e55b1aa60bdfd6"}},{"code_sha256_prefix":"c5c095f2793b10c6","entry":"build_mapping_trie","repo":"Mikki99/ipa-tokenization","repo_kind":"found_in_text","path":"src/ipa_tok/g2p/jap_fix.py","file_url":"https://github.com/Mikki99/ipa-tokenization/blob/HEAD/src/ipa_tok/g2p/jap_fix.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c5c095f2793b10c6"}},{"code_sha256_prefix":"5b0c48e052484bfa","entry":"build_trie","repo":"Mikki99/ipa-tokenization","repo_kind":"found_in_text","path":"src/ipa_tok/g2p/ipa_trie.py","file_url":"https://github.com/Mikki99/ipa-tokenization/blob/HEAD/src/ipa_tok/g2p/ipa_trie.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5b0c48e052484bfa"}},{"code_sha256_prefix":"920464019d7296ca","entry":"compute_renyi_entropies","repo":"Mikki99/ipa-tokenization","repo_kind":"found_in_text","path":"src/ipa_tok/eval/intrinsic.py","file_url":"https://github.com/Mikki99/ipa-tokenization/blob/HEAD/src/ipa_tok/eval/intrinsic.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"920464019d7296ca"}},{"code_sha256_prefix":"c645258c69182925","entry":"count_lines","repo":"Mikki99/ipa-tokenization","repo_kind":"found_in_text","path":"src/ipa_tok/gpt2/cache.py","file_url":"https://github.com/Mikki99/ipa-tokenization/blob/HEAD/src/ipa_tok/gpt2/cache.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c645258c69182925"}},{"code_sha256_prefix":"b3ee2e7fcfb924b6","entry":"find_longest_match","repo":"Mikki99/ipa-tokenization","repo_kind":"found_in_text","path":"src/ipa_tok/g2p/ipa_trie.py","file_url":"https://github.com/Mikki99/ipa-tokenization/blob/HEAD/src/ipa_tok/g2p/ipa_trie.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b3ee2e7fcfb924b6"}},{"code_sha256_prefix":"7a19210e4c723bce","entry":"is_ignored_char","repo":"Mikki99/ipa-tokenization","repo_kind":"found_in_text","path":"src/ipa_tok/g2p/ipa_trie.py","file_url":"https://github.com/Mikki99/ipa-tokenization/blob/HEAD/src/ipa_tok/g2p/ipa_trie.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7a19210e4c723bce"}},{"code_sha256_prefix":"16d15bb4e9564ac4","entry":"json_load","repo":"Mikki99/ipa-tokenization","repo_kind":"found_in_text","path":"src/ipa_tok/eval/ranking.py","file_url":"https://github.com/Mikki99/ipa-tokenization/blob/HEAD/src/ipa_tok/eval/ranking.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"16d15bb4e9564ac4"}},{"code_sha256_prefix":"6ef5254cfdf45db9","entry":"load_ipa_set","repo":"Mikki99/ipa-tokenization","repo_kind":"found_in_text","path":"src/ipa_tok/g2p/ipa_set.py","file_url":"https://github.com/Mikki99/ipa-tokenization/blob/HEAD/src/ipa_tok/g2p/ipa_set.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6ef5254cfdf45db9"}},{"code_sha256_prefix":"5a64f6f4a1b6920e","entry":"metric_results_path_for_tokenizer","repo":"Mikki99/ipa-tokenization","repo_kind":"found_in_text","path":"src/ipa_tok/eval/paths.py","file_url":"https://github.com/Mikki99/ipa-tokenization/blob/HEAD/src/ipa_tok/eval/paths.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5a64f6f4a1b6920e"}},{"code_sha256_prefix":"26e50accd3e62115","entry":"parse_tokenizer_id_from_model_path","repo":"Mikki99/ipa-tokenization","repo_kind":"found_in_text","path":"src/ipa_tok/eval/paths.py","file_url":"https://github.com/Mikki99/ipa-tokenization/blob/HEAD/src/ipa_tok/eval/paths.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"26e50accd3e62115"}},{"code_sha256_prefix":"7fa9a9d2ef998b50","entry":"reconstruct_characters_from_bytes","repo":"Mikki99/ipa-tokenization","repo_kind":"found_in_text","path":"src/ipa_tok/eval/intrinsic.py","file_url":"https://github.com/Mikki99/ipa-tokenization/blob/HEAD/src/ipa_tok/eval/intrinsic.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7fa9a9d2ef998b50"}},{"code_sha256_prefix":"e7b86614beeddd58","entry":"split_indices","repo":"Mikki99/ipa-tokenization","repo_kind":"found_in_text","path":"src/ipa_tok/gpt2/cache.py","file_url":"https://github.com/Mikki99/ipa-tokenization/blob/HEAD/src/ipa_tok/gpt2/cache.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e7b86614beeddd58"}},{"code_sha256_prefix":"239df1b0cbcbb98a","entry":"stable_seed","repo":"Mikki99/ipa-tokenization","repo_kind":"found_in_text","path":"src/ipa_tok/gpt2/cache.py","file_url":"https://github.com/Mikki99/ipa-tokenization/blob/HEAD/src/ipa_tok/gpt2/cache.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"239df1b0cbcbb98a"}},{"code_sha256_prefix":"cce9cf46aa219b0d","entry":"summarise_tokenizer_result","repo":"Mikki99/ipa-tokenization","repo_kind":"found_in_text","path":"src/ipa_tok/eval/ranking.py","file_url":"https://github.com/Mikki99/ipa-tokenization/blob/HEAD/src/ipa_tok/eval/ranking.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cce9cf46aa219b0d"}},{"code_sha256_prefix":"8816e85d176ceb85","entry":"resolve_punctuation_marks","repo":"Mikki99/ipa-tokenization","repo_kind":"found_in_text","path":"src/ipa_tok/g2p/phonemizer_convert.py","file_url":"https://github.com/Mikki99/ipa-tokenization/blob/HEAD/src/ipa_tok/g2p/phonemizer_convert.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8816e85d176ceb85"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.CL","source":"arxiv_2026.jsonl"},"syntology_extracted_results":null}