{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/an-empirical-study-on-cross-lingual","title":"An Empirical Study on Cross-lingual Vocabulary Adaptation for Efficient Language Model Inference","arxiv_id":"2402.10712","date":"2024-02-16","proceeding":null,"authors":["Atsuki Yamaguchi","Aline Villavicencio","Nikolaos Aletras"],"abstract":"The development of state-of-the-art generative large language models (LLMs) disproportionately relies on English-centric tokenizers, vocabulary and pre-training data. Despite the fact that some LLMs have multilingual capabilities, recent studies have shown that their inference efficiency deteriorates when generating text in languages other than English. This results in increased inference time and costs. Cross-lingual vocabulary adaptation (CVA) methods have been proposed for adapting models to a target language aiming to improve downstream performance. However, the effectiveness of these methods on increasing inference efficiency of generative LLMs has yet to be explored. In this paper, we perform an empirical study of five CVA methods on four generative LLMs (including monolingual and multilingual models) across four typologically-diverse languages and four natural language understanding tasks. We find that CVA substantially contributes to LLM inference speedups of up to 271.5\\%. We also show that adapting LLMs that have been pre-trained on more balanced multilingual data results in downstream performance comparable to the original models.","url_abs":"https://arxiv.org/abs/2402.10712v3","url_pdf":"https://arxiv.org/pdf/2402.10712v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"an-empirical-study-on-cross-lingual","repo_url":"https://github.com/gucci-j/llm-cva","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"natural-language-understanding","task_name":"Natural Language Understanding"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2402.10712","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.10712"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/gucci-j/llm-cva","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":5,"unverified":2},"by_repo_kind":{"official":{"samples":7,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"cfba8ed4574f17f5","entry":"filter_fn","repo":"gucci-j/llm-cva","repo_kind":"official","path":"preprocessing/src/train_fasttext.py","file_url":"https://github.com/gucci-j/llm-cva/blob/HEAD/preprocessing/src/train_fasttext.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cfba8ed4574f17f5"}},{"code_sha256_prefix":"788bf89500c4f4dc","entry":"filter_fn_cc100","repo":"gucci-j/llm-cva","repo_kind":"official","path":"preprocessing/src/preprocess.py","file_url":"https://github.com/gucci-j/llm-cva/blob/HEAD/preprocessing/src/preprocess.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"788bf89500c4f4dc"}},{"code_sha256_prefix":"d2183c85747240d9","entry":"group_texts","repo":"gucci-j/llm-cva","repo_kind":"official","path":"preprocessing/src/preprocess.py","file_url":"https://github.com/gucci-j/llm-cva/blob/HEAD/preprocessing/src/preprocess.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d2183c85747240d9"}},{"code_sha256_prefix":"03001067f37ba412","entry":"hex2dec","repo":"gucci-j/llm-cva","repo_kind":"official","path":"adaptation/src/core/heuristics_init.py","file_url":"https://github.com/gucci-j/llm-cva/blob/HEAD/adaptation/src/core/heuristics_init.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"03001067f37ba412"}},{"code_sha256_prefix":"5fafc667b826a8d8","entry":"top_script","repo":"gucci-j/llm-cva","repo_kind":"official","path":"adaptation/src/core/heuristics_init.py","file_url":"https://github.com/gucci-j/llm-cva/blob/HEAD/adaptation/src/core/heuristics_init.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5fafc667b826a8d8"}},{"code_sha256_prefix":"e89d93b6bfdc0cde","entry":"get_ord2script","repo":"gucci-j/llm-cva","repo_kind":"official","path":"adaptation/src/core/heuristics_init.py","file_url":"https://github.com/gucci-j/llm-cva/blob/HEAD/adaptation/src/core/heuristics_init.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e89d93b6bfdc0cde"}},{"code_sha256_prefix":"cee6083e09139c9a","entry":"round_to_nearest_multiple","repo":"gucci-j/llm-cva","repo_kind":"official","path":"adaptation/src/core/clp_plus_init.py","file_url":"https://github.com/gucci-j/llm-cva/blob/HEAD/adaptation/src/core/clp_plus_init.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cee6083e09139c9a"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}