{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/meta-learning-the-difference-preparing-large","title":"Meta-Learning the Difference: Preparing Large Language Models for Efficient Adaptation","arxiv_id":"2207.03509","date":"2022-07-07","proceeding":null,"authors":["Zejiang Hou","Julian Salazar","George Polovets"],"abstract":"Large pretrained language models (PLMs) are often domain- or task-adapted via fine-tuning or prompting. Finetuning requires modifying all of the parameters and having enough data to avoid overfitting while prompting requires no training and few examples but limits performance. Instead, we prepare PLMs for data- and parameter-efficient adaptation by learning to learn the difference between general and adapted PLMs. This difference is expressed in terms of model weights and sublayer structure through our proposed dynamic low-rank reparameterization and learned architecture controller. Experiments on few-shot dialogue completion, low-resource abstractive summarization, and multi-domain language modeling show improvements in adaptation time and performance over direct finetuning or preparation via domain-adaptive pretraining. Ablations show our task-adaptive reparameterization (TARP) and model search (TAMS) components individually improve on other parameter-efficient transfer like adapters and structure-learning methods like learned sparsification.","url_abs":"https://arxiv.org/abs/2207.03509v1","url_pdf":"https://arxiv.org/pdf/2207.03509v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"meta-learning-the-difference-preparing-large","repo_url":"https://github.com/amazon-research/meta-learning-the-difference","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"abstractive-text-summarization","task_name":"Abstractive Text Summarization"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"meta-learning","task_name":"Meta-Learning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2207.03509","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2207.03509"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/amazon-research/meta-learning-the-difference","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_draft_wrong":1,"unverified":7},"by_repo_kind":{"official":{"samples":8,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"a315b8fc32a18efe","entry":"shift_tokens_right","repo":"amazon-research/meta-learning-the-difference","repo_kind":"official","path":"abstractive_summarization/src/modeling_bart.py","file_url":"https://github.com/amazon-research/meta-learning-the-difference/blob/HEAD/abstractive_summarization/src/modeling_bart.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a315b8fc32a18efe"}},{"code_sha256_prefix":"ab3383a6a23c16e7","entry":"add_noise","repo":"amazon-research/meta-learning-the-difference","repo_kind":"official","path":"abstractive_summarization/src/dapt_pretraining.py","file_url":"https://github.com/amazon-research/meta-learning-the-difference/blob/HEAD/abstractive_summarization/src/dapt_pretraining.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ab3383a6a23c16e7"}},{"code_sha256_prefix":"181f336b2ac65f8a","entry":"get_task_embedding","repo":"amazon-research/meta-learning-the-difference","repo_kind":"official","path":"abstractive_summarization/src/trainer.py","file_url":"https://github.com/amazon-research/meta-learning-the-difference/blob/HEAD/abstractive_summarization/src/trainer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"181f336b2ac65f8a"}},{"code_sha256_prefix":"e1ace7b67d859683","entry":"make_file_name","repo":"amazon-research/meta-learning-the-difference","repo_kind":"official","path":"abstractive_summarization/src/trainer.py","file_url":"https://github.com/amazon-research/meta-learning-the-difference/blob/HEAD/abstractive_summarization/src/trainer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e1ace7b67d859683"}},{"code_sha256_prefix":"3aa756a41c020da7","entry":"rouge_results_to_str","repo":"amazon-research/meta-learning-the-difference","repo_kind":"official","path":"abstractive_summarization/src/cal_rouge.py","file_url":"https://github.com/amazon-research/meta-learning-the-difference/blob/HEAD/abstractive_summarization/src/cal_rouge.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3aa756a41c020da7"}},{"code_sha256_prefix":"3987b07f6a36ed6f","entry":"sent_permutation","repo":"amazon-research/meta-learning-the-difference","repo_kind":"official","path":"abstractive_summarization/src/dapt_pretraining.py","file_url":"https://github.com/amazon-research/meta-learning-the-difference/blob/HEAD/abstractive_summarization/src/dapt_pretraining.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3987b07f6a36ed6f"}},{"code_sha256_prefix":"8a9e45524c17becc","entry":"text_infilling","repo":"amazon-research/meta-learning-the-difference","repo_kind":"official","path":"abstractive_summarization/src/dapt_pretraining.py","file_url":"https://github.com/amazon-research/meta-learning-the-difference/blob/HEAD/abstractive_summarization/src/dapt_pretraining.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8a9e45524c17becc"}},{"code_sha256_prefix":"966f63a47c20ae79","entry":"tokenize","repo":"amazon-research/meta-learning-the-difference","repo_kind":"official","path":"abstractive_summarization/src/inference.py","file_url":"https://github.com/amazon-research/meta-learning-the-difference/blob/HEAD/abstractive_summarization/src/inference.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"966f63a47c20ae79"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}