{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/lora-efficient-low-rank-adaptation-of-large","title":"LoRA+: Efficient Low Rank Adaptation of Large Models","arxiv_id":"2402.12354","date":"2024-02-19","proceeding":null,"authors":["Soufiane Hayou","Nikhil Ghosh","Bin Yu"],"abstract":"In this paper, we show that Low Rank Adaptation (LoRA) as originally introduced in Hu et al. (2021) leads to suboptimal finetuning of models with large width (embedding dimension). This is due to the fact that adapter matrices A and B in LoRA are updated with the same learning rate. Using scaling arguments for large width networks, we demonstrate that using the same learning rate for A and B does not allow efficient feature learning. We then show that this suboptimality of LoRA can be corrected simply by setting different learning rates for the LoRA adapter matrices A and B with a well-chosen ratio. We call this proposed algorithm LoRA$+$. In our extensive experiments, LoRA$+$ improves performance (1-2 $\\%$ improvements) and finetuning speed (up to $\\sim$ 2X SpeedUp), at the same computational cost as LoRA.","url_abs":"https://arxiv.org/abs/2402.12354v2","url_pdf":"https://arxiv.org/pdf/2402.12354v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"lora-efficient-low-rank-adaptation-of-large","repo_url":"https://github.com/nikhil-ghosh-berkeley/loraplus","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"lora-efficient-low-rank-adaptation-of-large","repo_url":"https://github.com/seanzhang-zhichen/llama3-chinese","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[],"methods":[{"method_slug":"adapter","method_name":"Adapter"},{"method_slug":"speed","method_name":"SPEED"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2402.12354","atlas_url":"https://app.syntology.ai/?focus=2402.12354","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.12354"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/nikhil-ghosh-berkeley/loraplus","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/seanzhang-zhichen/llama3-chinese","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"unverified":4},"by_repo_kind":{"official":{"samples":4,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"70ab4a45d0c68e3e","entry":"attempt_train","repo":"nikhil-ghosh-berkeley/loraplus","repo_kind":"official","path":"glue/src/train_utils.py","file_url":"https://github.com/nikhil-ghosh-berkeley/loraplus/blob/HEAD/glue/src/train_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"70ab4a45d0c68e3e"}},{"code_sha256_prefix":"e5ef69b1e4b7a75a","entry":"create_loraplus_optimizer","repo":"nikhil-ghosh-berkeley/loraplus","repo_kind":"official","path":"lora_plus.py","file_url":"https://github.com/nikhil-ghosh-berkeley/loraplus/blob/HEAD/lora_plus.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e5ef69b1e4b7a75a"}},{"code_sha256_prefix":"00c123899f8febfb","entry":"find_valid_checkpoint","repo":"nikhil-ghosh-berkeley/loraplus","repo_kind":"official","path":"glue/src/train_utils.py","file_url":"https://github.com/nikhil-ghosh-berkeley/loraplus/blob/HEAD/glue/src/train_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"00c123899f8febfb"}},{"code_sha256_prefix":"ef1ba16faaacbb49","entry":"get_module","repo":"nikhil-ghosh-berkeley/loraplus","repo_kind":"official","path":"lora_plus.py","file_url":"https://github.com/nikhil-ghosh-berkeley/loraplus/blob/HEAD/lora_plus.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ef1ba16faaacbb49"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}