{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/the-expressive-power-of-low-rank-adaptation","title":"The Expressive Power of Low-Rank Adaptation","arxiv_id":"2310.17513","date":"2023-10-26","proceeding":null,"authors":["Yuchen Zeng","Kangwook Lee"],"abstract":"Low-Rank Adaptation (LoRA), a parameter-efficient fine-tuning method that leverages low-rank adaptation of weight matrices, has emerged as a prevalent technique for fine-tuning pre-trained models such as large language models and diffusion models. Despite its huge success in practice, the theoretical underpinnings of LoRA have largely remained unexplored. This paper takes the first step to bridge this gap by theoretically analyzing the expressive power of LoRA. We prove that, for fully connected neural networks, LoRA can adapt any model $f$ to accurately represent any smaller target model $\\overline{f}$ if LoRA-rank $\\geq(\\text{width of }f) \\times \\frac{\\text{depth of }\\overline{f}}{\\text{depth of }f}$. We also quantify the approximation error when LoRA-rank is lower than the threshold. For Transformer networks, we show any model can be adapted to a target model of the same size with rank-$(\\frac{\\text{embedding size}}{2})$ LoRA adapters.","url_abs":"https://arxiv.org/abs/2310.17513v3","url_pdf":"https://arxiv.org/pdf/2310.17513v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"the-expressive-power-of-low-rank-adaptation","repo_url":"https://github.com/uw-madison-lee-lab/expressive_power_of_lora","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"jax","reach":{"status":"ok"}}],"tasks":[{"task_slug":"parameter-efficient-fine-tuning","task_name":"parameter-efficient fine-tuning"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"diffusion","method_name":"Diffusion"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2310.17513","atlas_url":"https://app.syntology.ai/?focus=2310.17513","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.17513"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/uw-madison-lee-lab/expressive_power_of_lora","reach":{"status":"ok"}}],"summary":{"ran":5},"by_repo_kind":{"official":{"samples":5,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":5,"samples":[{"code_sha256_prefix":"36b1a24d8c90e627","entry":"generate_diag_matrix","repo":"uw-madison-lee-lab/expressive_power_of_lora","repo_kind":"official","path":"helper.py","file_url":"https://github.com/uw-madison-lee-lab/expressive_power_of_lora/blob/HEAD/helper.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"36b1a24d8c90e627"}},{"code_sha256_prefix":"0145929644b22528","entry":"get_rows","repo":"uw-madison-lee-lab/expressive_power_of_lora","repo_kind":"official","path":"configs/failed_config.py","file_url":"https://github.com/uw-madison-lee-lab/expressive_power_of_lora/blob/HEAD/configs/failed_config.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0145929644b22528"}},{"code_sha256_prefix":"660073ae6862ac6e","entry":"lora_state_dict","repo":"uw-madison-lee-lab/expressive_power_of_lora","repo_kind":"official","path":"references/loralib/utils.py","file_url":"https://github.com/uw-madison-lee-lab/expressive_power_of_lora/blob/HEAD/references/loralib/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"660073ae6862ac6e"}},{"code_sha256_prefix":"85686a7f4d933809","entry":"my_int","repo":"uw-madison-lee-lab/expressive_power_of_lora","repo_kind":"official","path":"helper.py","file_url":"https://github.com/uw-madison-lee-lab/expressive_power_of_lora/blob/HEAD/helper.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"85686a7f4d933809"}},{"code_sha256_prefix":"7055a012d343e45a","entry":"our_construction","repo":"uw-madison-lee-lab/expressive_power_of_lora","repo_kind":"official","path":"helper.py","file_url":"https://github.com/uw-madison-lee-lab/expressive_power_of_lora/blob/HEAD/helper.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7055a012d343e45a"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}