{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/revisiting-weight-averaging-for-model-merging","title":"Revisiting Weight Averaging for Model Merging","arxiv_id":"2412.12153","date":"2024-12-11","proceeding":null,"authors":["Jiho Choi","Donggyun Kim","Chanhyuk Lee","Seunghoon Hong"],"abstract":"Model merging aims to build a multi-task learner by combining the parameters of individually fine-tuned models without additional training. While a straightforward approach is to average model parameters across tasks, this often results in suboptimal performance due to interference among parameters across tasks. In this paper, we present intriguing results that weight averaging implicitly induces task vectors centered around the weight averaging itself and that applying a low-rank approximation to these centered task vectors significantly improves merging performance. Our analysis shows that centering the task vectors effectively separates core task-specific knowledge and nuisance noise within the fine-tuned parameters into the top and lower singular vectors, respectively, allowing us to reduce inter-task interference through its low-rank approximation. We evaluate our method on eight image classification tasks, demonstrating that it outperforms prior methods by a significant margin, narrowing the performance gap with traditional multi-task learning to within 1-3%","url_abs":"https://arxiv.org/abs/2412.12153v1","url_pdf":"https://arxiv.org/pdf/2412.12153v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"revisiting-weight-averaging-for-model-merging","repo_url":"https://github.com/JH-GEECS/CART_public","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"revisiting-weight-averaging-for-model-merging","repo_url":"https://github.com/david3684/adarank","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"image-classification","task_name":"Image Classification"},{"task_slug":"multi-task-learning","task_name":"Multi-Task Learning"},{"task_slug":"image-classification","task_name":"image-classification"},{"task_slug":"model","task_name":"model"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2412.12153","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2412.12153"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/david3684/adarank","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/JH-GEECS/CART_public","reach":{"status":"ok"}}],"summary":{"ran":3,"ran_draft_wrong":2,"unverified":6},"by_repo_kind":{"official":{"samples":11,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":11,"samples":[{"code_sha256_prefix":"6d77ee840479c381","entry":"convert","repo":"JH-GEECS/CART_public","repo_kind":"official","path":"src/dataset/cifar10.py","file_url":"https://github.com/JH-GEECS/CART_public/blob/HEAD/src/dataset/cifar10.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6d77ee840479c381"}},{"code_sha256_prefix":"2d5d7a12d8de02b7","entry":"cosine_lr","repo":"JH-GEECS/CART_public","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/JH-GEECS/CART_public/blob/HEAD/src/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2d5d7a12d8de02b7"}},{"code_sha256_prefix":"b1e9e0d3d7d7e615","entry":"maybe_dictionarize","repo":"JH-GEECS/CART_public","repo_kind":"official","path":"src/dataset/common.py","file_url":"https://github.com/JH-GEECS/CART_public/blob/HEAD/src/dataset/common.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b1e9e0d3d7d7e615"}},{"code_sha256_prefix":"241a2a00ba0c307b","entry":"pretify_classname","repo":"JH-GEECS/CART_public","repo_kind":"official","path":"src/dataset/eurosat.py","file_url":"https://github.com/JH-GEECS/CART_public/blob/HEAD/src/dataset/eurosat.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"241a2a00ba0c307b"}},{"code_sha256_prefix":"c2df94ca3de67eb0","entry":"softmax_entropy","repo":"JH-GEECS/CART_public","repo_kind":"official","path":"src/merging.py","file_url":"https://github.com/JH-GEECS/CART_public/blob/HEAD/src/merging.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c2df94ca3de67eb0"}},{"code_sha256_prefix":"aafcff3ad1ccef6e","entry":"accuracy","repo":"JH-GEECS/CART_public","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/JH-GEECS/CART_public/blob/HEAD/src/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"aafcff3ad1ccef6e"}},{"code_sha256_prefix":"3b97c12751c0cf5b","entry":"get_features","repo":"JH-GEECS/CART_public","repo_kind":"official","path":"src/dataset/common.py","file_url":"https://github.com/JH-GEECS/CART_public/blob/HEAD/src/dataset/common.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3b97c12751c0cf5b"}},{"code_sha256_prefix":"238ff32fcbf410d6","entry":"get_features_helper","repo":"JH-GEECS/CART_public","repo_kind":"official","path":"src/dataset/common.py","file_url":"https://github.com/JH-GEECS/CART_public/blob/HEAD/src/dataset/common.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"238ff32fcbf410d6"}},{"code_sha256_prefix":"76cd8313a00d0d2c","entry":"load_config","repo":"JH-GEECS/CART_public","repo_kind":"official","path":"src/merging.py","file_url":"https://github.com/JH-GEECS/CART_public/blob/HEAD/src/merging.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"76cd8313a00d0d2c"}},{"code_sha256_prefix":"21332e5b7d6c16cc","entry":"straight_through_mask","repo":"JH-GEECS/CART_public","repo_kind":"official","path":"src/merging.py","file_url":"https://github.com/JH-GEECS/CART_public/blob/HEAD/src/merging.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"21332e5b7d6c16cc"}},{"code_sha256_prefix":"7d051a7adefedb6c","entry":"torch_load_old","repo":"JH-GEECS/CART_public","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/JH-GEECS/CART_public/blob/HEAD/src/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7d051a7adefedb6c"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}