{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/mixture-of-subspaces-in-low-rank-adaptation","title":"Mixture-of-Subspaces in Low-Rank Adaptation","arxiv_id":"2406.11909","date":"2024-06-16","proceeding":null,"authors":["Taiqiang Wu","Jiahao Wang","Zhe Zhao","Ngai Wong"],"abstract":"In this paper, we introduce a subspace-inspired Low-Rank Adaptation (LoRA) method, which is computationally efficient, easy to implement, and readily applicable to large language, multimodal, and diffusion models. Initially, we equivalently decompose the weights of LoRA into two subspaces, and find that simply mixing them can enhance performance. To study such a phenomenon, we revisit it through a fine-grained subspace lens, showing that such modification is equivalent to employing a fixed mixer to fuse the subspaces. To be more flexible, we jointly learn the mixer with the original LoRA weights, and term the method Mixture-of-Subspaces LoRA (MoSLoRA). MoSLoRA consistently outperforms LoRA on tasks in different modalities, including commonsense reasoning, visual instruction tuning, and subject-driven text-to-image generation, demonstrating its effectiveness and robustness. Codes are available at https://github.com/wutaiqiang/MoSLoRA.","url_abs":"https://arxiv.org/abs/2406.11909v3","url_pdf":"https://arxiv.org/pdf/2406.11909v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"mixture-of-subspaces-in-low-rank-adaptation","repo_url":"https://github.com/wutaiqiang/moslora","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"common-sense-reasoning","task_name":"Common Sense Reasoning"},{"task_slug":"image-generation","task_name":"Image Generation"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"sentence-completion","task_name":"Sentence Completion"},{"task_slug":"text-to-image-generation-1","task_name":"Text to Image Generation"},{"task_slug":"text-to-image-generation","task_name":"Text-to-Image Generation"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/common-sense-reasoning-on-arc-challenge","task":"Common Sense Reasoning","dataset":"ARC (Challenge)","model":"LLaMA 3 8B + MoSLoRA (fine-tuned)","rank_in_archive_order":16,"of":54,"metrics":{"Accuracy":"81.5"},"uses_additional_data":false},{"leaderboard":"/sota/common-sense-reasoning-on-arc-easy","task":"Common Sense Reasoning","dataset":"ARC (Easy)","model":"LLaMA 3 8B+MoSLoRA (fine-tuned)","rank_in_archive_order":2,"of":47,"metrics":{"Accuracy":"90.5"},"uses_additional_data":false},{"leaderboard":"/sota/common-sense-reasoning-on-winogrande","task":"Common Sense Reasoning","dataset":"WinoGrande","model":"LLaMA3 8B+MoSLoRA","rank_in_archive_order":10,"of":77,"metrics":{"Accuracy":"85.8"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-boolq","task":"Question Answering","dataset":"BoolQ","model":"LLaMA3+MoSLoRA","rank_in_archive_order":38,"of":65,"metrics":{"Accuracy":"74.6"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-openbookqa","task":"Question Answering","dataset":"OpenBookQA","model":"LLaMA-3 8B+MoSLoRA","rank_in_archive_order":14,"of":45,"metrics":{"Accuracy":"86.8"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-piqa","task":"Question Answering","dataset":"PIQA","model":"LLaMA3 8B+MoSLoRA","rank_in_archive_order":2,"of":67,"metrics":{"Accuracy":"89.7"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-social-iqa","task":"Question Answering","dataset":"SIQA","model":"LLaMA-3 8B+MoSLoRA (fine-tuned)","rank_in_archive_order":5,"of":24,"metrics":{"Accuracy":"81.0"},"uses_additional_data":false},{"leaderboard":"/sota/sentence-completion-on-hellaswag","task":"Sentence Completion","dataset":"HellaSwag","model":"LLaMA3+MoSLoRA","rank_in_archive_order":5,"of":89,"metrics":{"Accuracy":"95.0"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-mm-vet","task":"Visual Question Answering","dataset":"MM-Vet","model":"LLaVA-InternLM2-7B-ViT + MoSLoRA","rank_in_archive_order":159,"of":231,"metrics":{"GPT-4 score":"35.2"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-mm-vet","task":"Visual Question Answering","dataset":"MM-Vet","model":"InternLM2+ViT (QMoSLoRA)","rank_in_archive_order":160,"of":231,"metrics":{"GPT-4 score":"35.2"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-mmbench","task":"Visual Question Answering","dataset":"MMBench","model":"LLaVA-InternLM2-ViT + MoSLoRA","rank_in_archive_order":1,"of":5,"metrics":{"GPT-3.5 score":"73.8"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-mmbench","task":"Visual Question Answering","dataset":"MMBench","model":"LLaVA-LLaMA3-8B-ViT + MoSLoRA","rank_in_archive_order":3,"of":5,"metrics":{"GPT-3.5 score":"73.0"},"uses_additional_data":false}],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2406.11909","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.11909"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/wutaiqiang/moslora","reach":{"status":"ok"}}],"summary":{"ran":1,"ran_draft_wrong":3,"unverified":2},"by_repo_kind":{"official":{"samples":6,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":6,"samples":[{"code_sha256_prefix":"03e1c6795642a8db","entry":"determine_scheduler_type","repo":"wutaiqiang/moslora","repo_kind":"official","path":"subject_driven_generation/train_dreambooth_lora_sdxl.py","file_url":"https://github.com/wutaiqiang/moslora/blob/HEAD/subject_driven_generation/train_dreambooth_lora_sdxl.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"03e1c6795642a8db"}},{"code_sha256_prefix":"56e674752db00649","entry":"generate_prompt","repo":"wutaiqiang/moslora","repo_kind":"official","path":"commonsense_reasoning/finetune.py","file_url":"https://github.com/wutaiqiang/moslora/blob/HEAD/commonsense_reasoning/finetune.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"56e674752db00649"}},{"code_sha256_prefix":"1c6b8be890b6118a","entry":"generate_prompt","repo":"wutaiqiang/moslora","repo_kind":"official","path":"commonsense_reasoning/commonsense_evaluate.py","file_url":"https://github.com/wutaiqiang/moslora/blob/HEAD/commonsense_reasoning/commonsense_evaluate.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1c6b8be890b6118a"}},{"code_sha256_prefix":"56a9466ff35b35bb","entry":"load_data","repo":"wutaiqiang/moslora","repo_kind":"official","path":"commonsense_reasoning/commonsense_evaluate.py","file_url":"https://github.com/wutaiqiang/moslora/blob/HEAD/commonsense_reasoning/commonsense_evaluate.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"56a9466ff35b35bb"}},{"code_sha256_prefix":"86965b6bae051ea4","entry":"import_model_class_from_model_name_or_path","repo":"wutaiqiang/moslora","repo_kind":"official","path":"subject_driven_generation/train_dreambooth_lora_sdxl.py","file_url":"https://github.com/wutaiqiang/moslora/blob/HEAD/subject_driven_generation/train_dreambooth_lora_sdxl.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"86965b6bae051ea4"}},{"code_sha256_prefix":"b8da7d64c2442014","entry":"main","repo":"wutaiqiang/moslora","repo_kind":"official","path":"commonsense_reasoning/commonsense_evaluate.py","file_url":"https://github.com/wutaiqiang/moslora/blob/HEAD/commonsense_reasoning/commonsense_evaluate.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b8da7d64c2442014"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}