{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/recite-reconstruct-recollect-memorization-in","title":"Recite, Reconstruct, Recollect: Memorization in LMs as a Multifaceted Phenomenon","arxiv_id":"2406.17746","date":"2024-06-25","proceeding":null,"authors":["USVSN Sai Prashanth","Alvin Deng","Kyle O'Brien","Jyothir S V","Mohammad Aflah Khan","Jaydeep Borkar","Christopher A. Choquette-Choo","Jacob Ray Fuehne","Stella Biderman","Tracy Ke","Katherine Lee","Naomi Saphra"],"abstract":"Memorization in language models is typically treated as a homogenous phenomenon, neglecting the specifics of the memorized data. We instead model memorization as the effect of a set of complex factors that describe each sample and relate it to the model and corpus. To build intuition around these factors, we break memorization down into a taxonomy: recitation of highly duplicated sequences, reconstruction of inherently predictable sequences, and recollection of sequences that are neither. We demonstrate the usefulness of our taxonomy by using it to construct a predictive model for memorization. By analyzing dependencies and inspecting the weights of the predictive model, we find that different factors influence the likelihood of memorization differently depending on the taxonomic category.","url_abs":"https://arxiv.org/abs/2406.17746v2","url_pdf":"https://arxiv.org/pdf/2406.17746v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"recite-reconstruct-recollect-memorization-in","repo_url":"https://github.com/eleutherai/semantic-memorization","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"memorization","task_name":"Memorization"}],"methods":[{"method_slug":"set","method_name":"SET"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2406.17746","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.17746"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/eleutherai/semantic-memorization","reach":{"status":"ok"}}],"summary":{"ran":4,"unverified":2},"by_repo_kind":{"official":{"samples":6,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":6,"samples":[{"code_sha256_prefix":"f6beaa0ccde5ca7b","entry":"calculate_perplexity","repo":"eleutherai/semantic-memorization","repo_kind":"official","path":"inference.py","file_url":"https://github.com/eleutherai/semantic-memorization/blob/HEAD/inference.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f6beaa0ccde5ca7b"}},{"code_sha256_prefix":"a04b720114cec6f5","entry":"derive_is_templating_feature","repo":"eleutherai/semantic-memorization","repo_kind":"official","path":"model_parameters.py","file_url":"https://github.com/eleutherai/semantic-memorization/blob/HEAD/model_parameters.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a04b720114cec6f5"}},{"code_sha256_prefix":"736b3b9ab49b28d1","entry":"expected_calibration_error","repo":"eleutherai/semantic-memorization","repo_kind":"official","path":"model_utils.py","file_url":"https://github.com/eleutherai/semantic-memorization/blob/HEAD/model_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"736b3b9ab49b28d1"}},{"code_sha256_prefix":"5168d8641144741d","entry":"taxonomy_function","repo":"eleutherai/semantic-memorization","repo_kind":"official","path":"model_parameters.py","file_url":"https://github.com/eleutherai/semantic-memorization/blob/HEAD/model_parameters.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5168d8641144741d"}},{"code_sha256_prefix":"e6da393d365fbc8a","entry":"load_model","repo":"eleutherai/semantic-memorization","repo_kind":"official","path":"inference.py","file_url":"https://github.com/eleutherai/semantic-memorization/blob/HEAD/inference.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e6da393d365fbc8a"}},{"code_sha256_prefix":"a1952e4763d93dba","entry":"load_tokenizer","repo":"eleutherai/semantic-memorization","repo_kind":"official","path":"inference.py","file_url":"https://github.com/eleutherai/semantic-memorization/blob/HEAD/inference.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a1952e4763d93dba"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}