{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/lines-post-training-layer-scaling-prevents","title":"LiNeS: Post-training Layer Scaling Prevents Forgetting and Enhances Model Merging","arxiv_id":"2410.17146","date":"2024-10-22","proceeding":null,"authors":["Ke Wang","Nikolaos Dimitriadis","Alessandro Favero","Guillermo Ortiz-Jimenez","Francois Fleuret","Pascal Frossard"],"abstract":"Large pre-trained models exhibit impressive zero-shot performance across diverse tasks, but fine-tuning often leads to catastrophic forgetting, where improvements on a target domain degrade generalization on other tasks. To address this challenge, we introduce LiNeS, Layer-increasing Network Scaling, a post-training editing technique designed to preserve pre-trained generalization while enhancing fine-tuned task performance. LiNeS scales parameter updates linearly based on their layer depth within the network, maintaining shallow layers close to their pre-trained values to preserve general features while allowing deeper layers to retain task-specific representations. We further extend this approach to multi-task model merging scenarios, where layer-wise scaling of merged parameters reduces negative task interference. LiNeS demonstrates significant improvements in both single-task and multi-task settings across various benchmarks in vision and natural language processing. It mitigates forgetting, enhances out-of-distribution generalization, integrates seamlessly with existing multi-task model merging baselines improving their performance across benchmarks and model sizes, and can boost generalization when merging LLM policies aligned with different rewards via RLHF. Importantly, our method is simple to implement and complementary to many existing techniques.","url_abs":"https://arxiv.org/abs/2410.17146v1","url_pdf":"https://arxiv.org/pdf/2410.17146v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"lines-post-training-layer-scaling-prevents","repo_url":"https://github.com/wang-kee/lines","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"out-of-distribution-generalization","task_name":"Out-of-Distribution Generalization"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2410.17146","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.17146"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/wang-kee/LiNeS","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/wang-kee/lines","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":2,"ran_draft_wrong":1,"unverified":3},"by_repo_kind":{"official":{"samples":6,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"6d77ee840479c381","entry":"convert","repo":"wang-kee/LiNeS","repo_kind":"official","path":"src/datasets/cifar10.py","file_url":"https://github.com/wang-kee/LiNeS/blob/HEAD/src/datasets/cifar10.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6d77ee840479c381"}},{"code_sha256_prefix":"b1e9e0d3d7d7e615","entry":"maybe_dictionarize","repo":"wang-kee/LiNeS","repo_kind":"official","path":"src/datasets/common.py","file_url":"https://github.com/wang-kee/LiNeS/blob/HEAD/src/datasets/common.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b1e9e0d3d7d7e615"}},{"code_sha256_prefix":"241a2a00ba0c307b","entry":"pretify_classname","repo":"wang-kee/LiNeS","repo_kind":"official","path":"src/datasets/eurosat.py","file_url":"https://github.com/wang-kee/LiNeS/blob/HEAD/src/datasets/eurosat.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"241a2a00ba0c307b"}},{"code_sha256_prefix":"10c1e5b6d05ea3f2","entry":"_TaskVector","repo":"wang-kee/lines","repo_kind":"official","path":"src/models/task_vectors.py","file_url":"https://github.com/wang-kee/lines/blob/HEAD/src/models/task_vectors.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"10c1e5b6d05ea3f2"}},{"code_sha256_prefix":"3b97c12751c0cf5b","entry":"get_features","repo":"wang-kee/LiNeS","repo_kind":"official","path":"src/datasets/common.py","file_url":"https://github.com/wang-kee/LiNeS/blob/HEAD/src/datasets/common.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3b97c12751c0cf5b"}},{"code_sha256_prefix":"238ff32fcbf410d6","entry":"get_features_helper","repo":"wang-kee/LiNeS","repo_kind":"official","path":"src/datasets/common.py","file_url":"https://github.com/wang-kee/LiNeS/blob/HEAD/src/datasets/common.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"238ff32fcbf410d6"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}