{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/hydralora-an-asymmetric-lora-architecture-for","title":"HydraLoRA: An Asymmetric LoRA Architecture for Efficient Fine-Tuning","arxiv_id":"2404.19245","date":"2024-04-30","proceeding":null,"authors":["Chunlin Tian","Zhan Shi","Zhijiang Guo","Li Li","Chengzhong Xu"],"abstract":"Adapting Large Language Models (LLMs) to new tasks through fine-tuning has been made more efficient by the introduction of Parameter-Efficient Fine-Tuning (PEFT) techniques, such as LoRA. However, these methods often underperform compared to full fine-tuning, particularly in scenarios involving complex datasets. This issue becomes even more pronounced in complex domains, highlighting the need for improved PEFT approaches that can achieve better performance. Through a series of experiments, we have uncovered two critical insights that shed light on the training and parameter inefficiency of LoRA. Building on these insights, we have developed HydraLoRA, a LoRA framework with an asymmetric structure that eliminates the need for domain expertise. Our experiments demonstrate that HydraLoRA outperforms other PEFT approaches, even those that rely on domain knowledge during the training and inference phases.","url_abs":"https://arxiv.org/abs/2404.19245v2","url_pdf":"https://arxiv.org/pdf/2404.19245v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"hydralora-an-asymmetric-lora-architecture-for","repo_url":"https://github.com/clin0212/hydralora","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"parameter-efficient-fine-tuning","task_name":"parameter-efficient fine-tuning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2404.19245","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.19245"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/clin0212/hydralora","reach":{"status":"ok"}}],"summary":{"ran":9,"ran_draft_wrong":1,"unverified":6},"by_repo_kind":{"official":{"samples":16,"ran":10,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":16,"samples":[{"code_sha256_prefix":"743ceb9025a7955d","entry":"booleans_processing","repo":"clin0212/hydralora","repo_kind":"official","path":"HydraLoRA/transformers/modeling_tf_utils.py","file_url":"https://github.com/clin0212/hydralora/blob/HEAD/HydraLoRA/transformers/modeling_tf_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"743ceb9025a7955d"}},{"code_sha256_prefix":"5168a172bbb363a5","entry":"convert_pytorch_state_dict_to_flax","repo":"clin0212/hydralora","repo_kind":"official","path":"HydraLoRA/transformers/modeling_flax_pytorch_utils.py","file_url":"https://github.com/clin0212/hydralora/blob/HEAD/HydraLoRA/transformers/modeling_flax_pytorch_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5168a172bbb363a5"}},{"code_sha256_prefix":"3c1eb64dde6b86af","entry":"dummy_loss","repo":"clin0212/hydralora","repo_kind":"official","path":"HydraLoRA/transformers/modeling_tf_utils.py","file_url":"https://github.com/clin0212/hydralora/blob/HEAD/HydraLoRA/transformers/modeling_tf_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3c1eb64dde6b86af"}},{"code_sha256_prefix":"ccaa0edba373622c","entry":"get_imports","repo":"clin0212/hydralora","repo_kind":"official","path":"HydraLoRA/transformers/dynamic_module_utils.py","file_url":"https://github.com/clin0212/hydralora/blob/HEAD/HydraLoRA/transformers/dynamic_module_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ccaa0edba373622c"}},{"code_sha256_prefix":"dd8f7751ef859581","entry":"infer_metric_tags_from_eval_results","repo":"clin0212/hydralora","repo_kind":"official","path":"HydraLoRA/transformers/modelcard.py","file_url":"https://github.com/clin0212/hydralora/blob/HEAD/HydraLoRA/transformers/modelcard.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"dd8f7751ef859581"}},{"code_sha256_prefix":"6a3959a0ffd36577","entry":"keras_serializable","repo":"clin0212/hydralora","repo_kind":"official","path":"HydraLoRA/transformers/modeling_tf_utils.py","file_url":"https://github.com/clin0212/hydralora/blob/HEAD/HydraLoRA/transformers/modeling_tf_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6a3959a0ffd36577"}},{"code_sha256_prefix":"a4491b45d19b7a39","entry":"parse_keras_history","repo":"clin0212/hydralora","repo_kind":"official","path":"HydraLoRA/transformers/modelcard.py","file_url":"https://github.com/clin0212/hydralora/blob/HEAD/HydraLoRA/transformers/modelcard.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a4491b45d19b7a39"}},{"code_sha256_prefix":"591cff6b55ba6bfb","entry":"prepare_model_for_kbit_training","repo":"clin0212/hydralora","repo_kind":"official","path":"HydraLoRA/fine-tuning.py","file_url":"https://github.com/clin0212/hydralora/blob/HEAD/HydraLoRA/fine-tuning.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"591cff6b55ba6bfb"}},{"code_sha256_prefix":"a66d2b78670c85e1","entry":"quick_gelu","repo":"clin0212/hydralora","repo_kind":"official","path":"HydraLoRA/transformers/modeling_flax_utils.py","file_url":"https://github.com/clin0212/hydralora/blob/HEAD/HydraLoRA/transformers/modeling_flax_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a66d2b78670c85e1"}},{"code_sha256_prefix":"61779b0406a59cfc","entry":"rename_key_and_reshape_tensor","repo":"clin0212/hydralora","repo_kind":"official","path":"HydraLoRA/transformers/modeling_flax_pytorch_utils.py","file_url":"https://github.com/clin0212/hydralora/blob/HEAD/HydraLoRA/transformers/modeling_flax_pytorch_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"61779b0406a59cfc"}},{"code_sha256_prefix":"2776dfe4cf53097d","entry":"build_instruction_dataset","repo":"clin0212/hydralora","repo_kind":"official","path":"HydraLoRA/build_dataset.py","file_url":"https://github.com/clin0212/hydralora/blob/HEAD/HydraLoRA/build_dataset.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2776dfe4cf53097d"}},{"code_sha256_prefix":"9d98806acb3144b7","entry":"dtype_byte_size","repo":"clin0212/hydralora","repo_kind":"official","path":"HydraLoRA/transformers/modeling_flax_utils.py","file_url":"https://github.com/clin0212/hydralora/blob/HEAD/HydraLoRA/transformers/modeling_flax_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9d98806acb3144b7"}},{"code_sha256_prefix":"e3ce70ca9258800e","entry":"flax_shard_checkpoint","repo":"clin0212/hydralora","repo_kind":"official","path":"HydraLoRA/transformers/modeling_flax_utils.py","file_url":"https://github.com/clin0212/hydralora/blob/HEAD/HydraLoRA/transformers/modeling_flax_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e3ce70ca9258800e"}},{"code_sha256_prefix":"00cc853a73f3aeb6","entry":"get_relative_import_files","repo":"clin0212/hydralora","repo_kind":"official","path":"HydraLoRA/transformers/dynamic_module_utils.py","file_url":"https://github.com/clin0212/hydralora/blob/HEAD/HydraLoRA/transformers/dynamic_module_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"00cc853a73f3aeb6"}},{"code_sha256_prefix":"450aeffe7f80f917","entry":"get_relative_imports","repo":"clin0212/hydralora","repo_kind":"official","path":"HydraLoRA/transformers/dynamic_module_utils.py","file_url":"https://github.com/clin0212/hydralora/blob/HEAD/HydraLoRA/transformers/dynamic_module_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"450aeffe7f80f917"}},{"code_sha256_prefix":"ec7b1528a936234d","entry":"is_hf_dataset","repo":"clin0212/hydralora","repo_kind":"official","path":"HydraLoRA/transformers/modelcard.py","file_url":"https://github.com/clin0212/hydralora/blob/HEAD/HydraLoRA/transformers/modelcard.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ec7b1528a936234d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}