{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/unveiling-linguistic-regions-in-large","title":"Unveiling Linguistic Regions in Large Language Models","arxiv_id":"2402.14700","date":"2024-02-22","proceeding":null,"authors":["Zhihao Zhang","Jun Zhao","Qi Zhang","Tao Gui","Xuanjing Huang"],"abstract":"Large Language Models (LLMs) have demonstrated considerable cross-lingual alignment and generalization ability. Current research primarily focuses on improving LLMs' cross-lingual generalization capabilities. However, there is still a lack of research on the intrinsic mechanisms of how LLMs achieve cross-lingual alignment. From the perspective of region partitioning, this paper conducts several investigations on the linguistic competence of LLMs. We discover a core region in LLMs that corresponds to linguistic competence, accounting for approximately 1% of the total model parameters. Removing this core region by setting parameters to zero results in a significant performance decrease across 30 different languages. Furthermore, this core region exhibits significant dimensional dependence, perturbations to even a single parameter on specific dimensions leading to a loss of linguistic competence. Moreover, we discover that distinct monolingual regions exist for different languages, and disruption to these specific regions substantially reduces the LLMs' proficiency in those corresponding languages. Our research also indicates that freezing the core linguistic region during further pre-training can mitigate the issue of catastrophic forgetting (CF), a common phenomenon observed during further pre-training of LLMs. Overall, exploring the LLMs' functional regions provides insights into the foundation of their intelligence.","url_abs":"https://arxiv.org/abs/2402.14700v3","url_pdf":"https://arxiv.org/pdf/2402.14700v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"unveiling-linguistic-regions-in-large","repo_url":"https://github.com/zzhang0179/Unveiling-Linguistic-Regions-in-LLMs","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2402.14700","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.14700"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/zzhang0179/Unveiling-Linguistic-Regions-in-LLMs","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":11,"unverified":2},"by_repo_kind":{"official":{"samples":13,"ran":11,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"e031eb012f38e70c","entry":"collate_fn_from_json","repo":"zzhang0179/Unveiling-Linguistic-Regions-in-LLMs","repo_kind":"official","path":"data_preprocess/preprocess-llama.py","file_url":"https://github.com/zzhang0179/Unveiling-Linguistic-Regions-in-LLMs/blob/HEAD/data_preprocess/preprocess-llama.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e031eb012f38e70c"}},{"code_sha256_prefix":"dd4433b03e7fa7bd","entry":"collate_fn_from_text","repo":"zzhang0179/Unveiling-Linguistic-Regions-in-LLMs","repo_kind":"official","path":"data_preprocess/preprocess-llama.py","file_url":"https://github.com/zzhang0179/Unveiling-Linguistic-Regions-in-LLMs/blob/HEAD/data_preprocess/preprocess-llama.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"dd4433b03e7fa7bd"}},{"code_sha256_prefix":"26d10e2041c3b7ae","entry":"compare_bool_matrix","repo":"zzhang0179/Unveiling-Linguistic-Regions-in-LLMs","repo_kind":"official","path":"region_selection/extract_accumulated_core_linguistic_region.py","file_url":"https://github.com/zzhang0179/Unveiling-Linguistic-Regions-in-LLMs/blob/HEAD/region_selection/extract_accumulated_core_linguistic_region.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"26d10e2041c3b7ae"}},{"code_sha256_prefix":"2797e4223999fe1e","entry":"count_lines","repo":"zzhang0179/Unveiling-Linguistic-Regions-in-LLMs","repo_kind":"official","path":"data_preprocess/preprocess-llama.py","file_url":"https://github.com/zzhang0179/Unveiling-Linguistic-Regions-in-LLMs/blob/HEAD/data_preprocess/preprocess-llama.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2797e4223999fe1e"}},{"code_sha256_prefix":"7f4bdf1898ad0544","entry":"get_eval_ds_config","repo":"zzhang0179/Unveiling-Linguistic-Regions-in-LLMs","repo_kind":"official","path":"training/utils/ds_utils.py","file_url":"https://github.com/zzhang0179/Unveiling-Linguistic-Regions-in-LLMs/blob/HEAD/training/utils/ds_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7f4bdf1898ad0544"}},{"code_sha256_prefix":"86613110e9a3611b","entry":"get_train_ds_config","repo":"zzhang0179/Unveiling-Linguistic-Regions-in-LLMs","repo_kind":"official","path":"training/utils/ds_utils.py","file_url":"https://github.com/zzhang0179/Unveiling-Linguistic-Regions-in-LLMs/blob/HEAD/training/utils/ds_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"86613110e9a3611b"}},{"code_sha256_prefix":"055c2b5ab3275ac2","entry":"jaccard_similarity","repo":"zzhang0179/Unveiling-Linguistic-Regions-in-LLMs","repo_kind":"official","path":"region_selection/extract_accumulated_core_linguistic_region.py","file_url":"https://github.com/zzhang0179/Unveiling-Linguistic-Regions-in-LLMs/blob/HEAD/region_selection/extract_accumulated_core_linguistic_region.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"055c2b5ab3275ac2"}},{"code_sha256_prefix":"0a2fbb705c3e7e85","entry":"logical_and_bool_matrix","repo":"zzhang0179/Unveiling-Linguistic-Regions-in-LLMs","repo_kind":"official","path":"region_selection/extract_accumulated_core_linguistic_region.py","file_url":"https://github.com/zzhang0179/Unveiling-Linguistic-Regions-in-LLMs/blob/HEAD/region_selection/extract_accumulated_core_linguistic_region.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0a2fbb705c3e7e85"}},{"code_sha256_prefix":"e617041853afd002","entry":"only_optimize_lora_parameters","repo":"zzhang0179/Unveiling-Linguistic-Regions-in-LLMs","repo_kind":"official","path":"training/utils/module/lora-old.py","file_url":"https://github.com/zzhang0179/Unveiling-Linguistic-Regions-in-LLMs/blob/HEAD/training/utils/module/lora-old.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e617041853afd002"}},{"code_sha256_prefix":"1e30b528dec22fba","entry":"set_first_false_to_true","repo":"zzhang0179/Unveiling-Linguistic-Regions-in-LLMs","repo_kind":"official","path":"training/step1_supervised_finetuning/accumulate_grad_mul_params_on_multi_languages.py","file_url":"https://github.com/zzhang0179/Unveiling-Linguistic-Regions-in-LLMs/blob/HEAD/training/step1_supervised_finetuning/accumulate_grad_mul_params_on_multi_languages.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1e30b528dec22fba"}},{"code_sha256_prefix":"6bdb682236e8d8bb","entry":"to_device","repo":"zzhang0179/Unveiling-Linguistic-Regions-in-LLMs","repo_kind":"official","path":"training/utils/utils.py","file_url":"https://github.com/zzhang0179/Unveiling-Linguistic-Regions-in-LLMs/blob/HEAD/training/utils/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6bdb682236e8d8bb"}},{"code_sha256_prefix":"02b56a01deeccf6b","entry":"get_all_reduce_mean","repo":"zzhang0179/Unveiling-Linguistic-Regions-in-LLMs","repo_kind":"official","path":"training/utils/utils.py","file_url":"https://github.com/zzhang0179/Unveiling-Linguistic-Regions-in-LLMs/blob/HEAD/training/utils/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"02b56a01deeccf6b"}},{"code_sha256_prefix":"17e20e9d289d2ca4","entry":"load_hf_tokenizer","repo":"zzhang0179/Unveiling-Linguistic-Regions-in-LLMs","repo_kind":"official","path":"training/utils/utils.py","file_url":"https://github.com/zzhang0179/Unveiling-Linguistic-Regions-in-LLMs/blob/HEAD/training/utils/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"17e20e9d289d2ca4"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}