{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/the-pitfalls-of-memorization-when","title":"The Pitfalls of Memorization: When Memorization Hurts Generalization","arxiv_id":"2412.07684","date":"2024-12-10","proceeding":null,"authors":["Reza Bayat","Mohammad Pezeshki","Elvis Dohmatob","David Lopez-Paz","Pascal Vincent"],"abstract":"Neural networks often learn simple explanations that fit the majority of the data while memorizing exceptions that deviate from these explanations.This behavior leads to poor generalization when the learned explanations rely on spurious correlations. In this work, we formalize the interplay between memorization and generalization, showing that spurious correlations would particularly lead to poor generalization when are combined with memorization. Memorization can reduce training loss to zero, leaving no incentive to learn robust, generalizable patterns. To address this, we propose memorization-aware training (MAT), which uses held-out predictions as a signal of memorization to shift a model's logits. MAT encourages learning robust patterns invariant across distributions, improving generalization under distribution shifts.","url_abs":"https://arxiv.org/abs/2412.07684v1","url_pdf":"https://arxiv.org/pdf/2412.07684v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"the-pitfalls-of-memorization-when","repo_url":"https://github.com/facebookresearch/pitfalls-of-memorization","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"memorization","task_name":"Memorization"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2412.07684","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2412.07684"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/pitfalls-of-memorization","reach":null}],"summary":{"ran_draft_wrong":4},"by_repo_kind":{"official":{"samples":4,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":4,"samples":[{"code_sha256_prefix":"77261461734a119a","entry":"get_data","repo":"facebookresearch/pitfalls-of-memorization","repo_kind":"official","path":"good_bad_ugly_memorization.py","file_url":"https://github.com/facebookresearch/pitfalls-of-memorization/blob/HEAD/good_bad_ugly_memorization.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"77261461734a119a"}},{"code_sha256_prefix":"8b34fc2919f528e3","entry":"problem_1","repo":"facebookresearch/pitfalls-of-memorization","repo_kind":"official","path":"interpretable_experiment.py","file_url":"https://github.com/facebookresearch/pitfalls-of-memorization/blob/HEAD/interpretable_experiment.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"8b34fc2919f528e3"}},{"code_sha256_prefix":"37e9b6781ed8ce73","entry":"run_erm","repo":"facebookresearch/pitfalls-of-memorization","repo_kind":"official","path":"interpretable_experiment.py","file_url":"https://github.com/facebookresearch/pitfalls-of-memorization/blob/HEAD/interpretable_experiment.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"37e9b6781ed8ce73"}},{"code_sha256_prefix":"d38da53e690d2d94","entry":"run_mat","repo":"facebookresearch/pitfalls-of-memorization","repo_kind":"official","path":"interpretable_experiment.py","file_url":"https://github.com/facebookresearch/pitfalls-of-memorization/blob/HEAD/interpretable_experiment.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"d38da53e690d2d94"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}