{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/localizing-paragraph-memorization-in-language","title":"Localizing Paragraph Memorization in Language Models","arxiv_id":"2403.19851","date":"2024-03-28","proceeding":null,"authors":["Niklas Stoehr","Mitchell Gordon","Chiyuan Zhang","Owen Lewis"],"abstract":"Can we localize the weights and mechanisms used by a language model to memorize and recite entire paragraphs of its training data? In this paper, we show that while memorization is spread across multiple layers and model components, gradients of memorized paragraphs have a distinguishable spatial pattern, being larger in lower model layers than gradients of non-memorized examples. Moreover, the memorized examples can be unlearned by fine-tuning only the high-gradient weights. We localize a low-layer attention head that appears to be especially involved in paragraph memorization. This head is predominantly focusing its attention on distinctive, rare tokens that are least frequent in a corpus-level unigram distribution. Next, we study how localized memorization is across the tokens in the prefix by perturbing tokens and measuring the caused change in the decoding. A few distinctive tokens early in a prefix can often corrupt the entire continuation. Overall, memorized continuations are not only harder to unlearn, but also to corrupt than non-memorized ones.","url_abs":"https://arxiv.org/abs/2403.19851v1","url_pdf":"https://arxiv.org/pdf/2403.19851v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"localizing-paragraph-memorization-in-language","repo_url":"https://github.com/googleinterns/localizing-paragraph-memorization","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"memorization","task_name":"Memorization"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2403.19851","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.19851"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/googleinterns/localizing-paragraph-memorization","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":4,"unverified":3},"by_repo_kind":{"official":{"samples":7,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"7f6795297ecde2c4","entry":"add_fwd_bwd_hooks","repo":"googleinterns/localizing-paragraph-memorization","repo_kind":"official","path":"utils/gradient.py","file_url":"https://github.com/googleinterns/localizing-paragraph-memorization/blob/HEAD/utils/gradient.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7f6795297ecde2c4"}},{"code_sha256_prefix":"39e32b875bb32e57","entry":"compute_exact_match","repo":"googleinterns/localizing-paragraph-memorization","repo_kind":"official","path":"utils/evaluation.py","file_url":"https://github.com/googleinterns/localizing-paragraph-memorization/blob/HEAD/utils/evaluation.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"39e32b875bb32e57"}},{"code_sha256_prefix":"5ea3cc284d436ea9","entry":"get_random_tok","repo":"googleinterns/localizing-paragraph-memorization","repo_kind":"official","path":"utils/patching.py","file_url":"https://github.com/googleinterns/localizing-paragraph-memorization/blob/HEAD/utils/patching.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5ea3cc284d436ea9"}},{"code_sha256_prefix":"a149ec1dd7e8a616","entry":"load_pile_splits","repo":"googleinterns/localizing-paragraph-memorization","repo_kind":"official","path":"utils/dataLoaders.py","file_url":"https://github.com/googleinterns/localizing-paragraph-memorization/blob/HEAD/utils/dataLoaders.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a149ec1dd7e8a616"}},{"code_sha256_prefix":"360137f938f59f88","entry":"find_topK_grads","repo":"googleinterns/localizing-paragraph-memorization","repo_kind":"official","path":"utils/intervening.py","file_url":"https://github.com/googleinterns/localizing-paragraph-memorization/blob/HEAD/utils/intervening.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"360137f938f59f88"}},{"code_sha256_prefix":"bf4474f40e39f1c8","entry":"load_pile_seqs","repo":"googleinterns/localizing-paragraph-memorization","repo_kind":"official","path":"utils/dataLoaders.py","file_url":"https://github.com/googleinterns/localizing-paragraph-memorization/blob/HEAD/utils/dataLoaders.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bf4474f40e39f1c8"}},{"code_sha256_prefix":"edb5d8f4ba6869f4","entry":"train_test_batching","repo":"googleinterns/localizing-paragraph-memorization","repo_kind":"official","path":"utils/dataLoaders.py","file_url":"https://github.com/googleinterns/localizing-paragraph-memorization/blob/HEAD/utils/dataLoaders.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"edb5d8f4ba6869f4"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}