{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/mora-high-rank-updating-for-parameter","title":"MoRA: High-Rank Updating for Parameter-Efficient Fine-Tuning","arxiv_id":"2405.12130","date":"2024-05-20","proceeding":null,"authors":["Ting Jiang","Shaohan Huang","Shengyue Luo","Zihan Zhang","Haizhen Huang","Furu Wei","Weiwei Deng","Feng Sun","Qi Zhang","Deqing Wang","Fuzhen Zhuang"],"abstract":"Low-rank adaptation is a popular parameter-efficient fine-tuning method for large language models. In this paper, we analyze the impact of low-rank updating, as implemented in LoRA. Our findings suggest that the low-rank updating mechanism may limit the ability of LLMs to effectively learn and memorize new knowledge. Inspired by this observation, we propose a new method called MoRA, which employs a square matrix to achieve high-rank updating while maintaining the same number of trainable parameters. To achieve it, we introduce the corresponding non-parameter operators to reduce the input dimension and increase the output dimension for the square matrix. Furthermore, these operators ensure that the weight can be merged back into LLMs, which makes our method can be deployed like LoRA. We perform a comprehensive evaluation of our method across five tasks: instruction tuning, mathematical reasoning, continual pretraining, memory and pretraining. Our method outperforms LoRA on memory-intensive tasks and achieves comparable performance on other tasks.","url_abs":"https://arxiv.org/abs/2405.12130v1","url_pdf":"https://arxiv.org/pdf/2405.12130v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"mora-high-rank-updating-for-parameter","repo_url":"https://github.com/kongds/mora","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"continual-pretraining","task_name":"Continual Pretraining"},{"task_slug":"mathematical-reasoning","task_name":"Mathematical Reasoning"},{"task_slug":"parameter-efficient-fine-tuning","task_name":"parameter-efficient fine-tuning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2405.12130","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.12130"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kongds/mora","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":1,"unverified":5},"by_repo_kind":{"official":{"samples":6,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"ecf94b05123201e3","entry":"bloom_model_postprocess_past_key_value","repo":"kongds/mora","repo_kind":"official","path":"peft-mora/src/peft/utils/constants.py","file_url":"https://github.com/kongds/mora/blob/HEAD/peft-mora/src/peft/utils/constants.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ecf94b05123201e3"}},{"code_sha256_prefix":"4b6850c585352061","entry":"check_adapters_to_merge","repo":"kongds/mora","repo_kind":"official","path":"peft-mora/src/peft/tuners/tuners_utils.py","file_url":"https://github.com/kongds/mora/blob/HEAD/peft-mora/src/peft/tuners/tuners_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4b6850c585352061"}},{"code_sha256_prefix":"c49fdcabd715ffef","entry":"check_target_module_exists","repo":"kongds/mora","repo_kind":"official","path":"peft-mora/src/peft/tuners/tuners_utils.py","file_url":"https://github.com/kongds/mora/blob/HEAD/peft-mora/src/peft/tuners/tuners_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c49fdcabd715ffef"}},{"code_sha256_prefix":"a3daaf3e5bdfc709","entry":"is_torch_tpu_available","repo":"kongds/mora","repo_kind":"official","path":"peft-mora/src/peft/import_utils.py","file_url":"https://github.com/kongds/mora/blob/HEAD/peft-mora/src/peft/import_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a3daaf3e5bdfc709"}},{"code_sha256_prefix":"9e5134c6d9aeba17","entry":"loftq_init","repo":"kongds/mora","repo_kind":"official","path":"peft-mora/src/peft/utils/loftq_utils.py","file_url":"https://github.com/kongds/mora/blob/HEAD/peft-mora/src/peft/utils/loftq_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9e5134c6d9aeba17"}},{"code_sha256_prefix":"249c10407ed14893","entry":"starcoder_model_postprocess_past_key_value","repo":"kongds/mora","repo_kind":"official","path":"peft-mora/src/peft/utils/constants.py","file_url":"https://github.com/kongds/mora/blob/HEAD/peft-mora/src/peft/utils/constants.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"249c10407ed14893"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}