{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/memory-layers-at-scale","title":"Memory Layers at Scale","arxiv_id":"2412.09764","date":"2024-12-12","proceeding":null,"authors":["Vincent-Pierre Berges","Barlas Oğuz","Daniel Haziza","Wen-tau Yih","Luke Zettlemoyer","Gargi Ghosh"],"abstract":"Memory layers use a trainable key-value lookup mechanism to add extra parameters to a model without increasing FLOPs. Conceptually, sparsely activated memory layers complement compute-heavy dense feed-forward layers, providing dedicated capacity to store and retrieve information cheaply. This work takes memory layers beyond proof-of-concept, proving their utility at contemporary scale. On downstream tasks, language models augmented with our improved memory layer outperform dense models with more than twice the computation budget, as well as mixture-of-expert models when matched for both compute and parameters. We find gains are especially pronounced for factual tasks. We provide a fully parallelizable memory layer implementation, demonstrating scaling laws with up to 128B memory parameters, pretrained to 1 trillion tokens, comparing to base models with up to 8B parameters.","url_abs":"https://arxiv.org/abs/2412.09764v2","url_pdf":"https://arxiv.org/pdf/2412.09764v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"memory-layers-at-scale","repo_url":"https://github.com/facebookresearch/memory","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}}],"tasks":[],"methods":[{"method_slug":"base","method_name":"BASE"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2412.09764","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2412.09764"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/memory","reach":{"status":"ok","spdx":"NOASSERTION"}}],"summary":{"ran":4,"unverified":15},"by_repo_kind":{"official":{"samples":19,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":19,"samples":[{"code_sha256_prefix":"e0ebc4027f1bd03b","entry":"dataclass_from_dict","repo":"facebookresearch/memory","repo_kind":"official","path":"lingua/args.py","file_url":"https://github.com/facebookresearch/memory/blob/HEAD/lingua/args.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e0ebc4027f1bd03b"}},{"code_sha256_prefix":"116b2f19f9a6148c","entry":"dataclass_to_dict","repo":"facebookresearch/memory","repo_kind":"official","path":"lingua/args.py","file_url":"https://github.com/facebookresearch/memory/blob/HEAD/lingua/args.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"116b2f19f9a6148c"}},{"code_sha256_prefix":"15be2e56e073a228","entry":"flatten_dict","repo":"facebookresearch/memory","repo_kind":"official","path":"lingua/args.py","file_url":"https://github.com/facebookresearch/memory/blob/HEAD/lingua/args.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"15be2e56e073a228"}},{"code_sha256_prefix":"bfdcca22e67a0a67","entry":"get_empty_buffer_state","repo":"facebookresearch/memory","repo_kind":"official","path":"lingua/data.py","file_url":"https://github.com/facebookresearch/memory/blob/HEAD/lingua/data.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bfdcca22e67a0a67"}},{"code_sha256_prefix":"3eb31c04dd0c9897","entry":"allclose","repo":"facebookresearch/memory","repo_kind":"official","path":"validate_distributed_embeddingbag.py","file_url":"https://github.com/facebookresearch/memory/blob/HEAD/validate_distributed_embeddingbag.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"3eb31c04dd0c9897"}},{"code_sha256_prefix":"be7ab0634167f84c","entry":"consolidate_checkpoints","repo":"facebookresearch/memory","repo_kind":"official","path":"lingua/checkpoint.py","file_url":"https://github.com/facebookresearch/memory/blob/HEAD/lingua/checkpoint.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"be7ab0634167f84c"}},{"code_sha256_prefix":"9caf44ff8e00b1dc","entry":"dist_max","repo":"facebookresearch/memory","repo_kind":"official","path":"lingua/distributed.py","file_url":"https://github.com/facebookresearch/memory/blob/HEAD/lingua/distributed.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9caf44ff8e00b1dc"}},{"code_sha256_prefix":"d09c7072d82024f4","entry":"dist_mean","repo":"facebookresearch/memory","repo_kind":"official","path":"lingua/distributed.py","file_url":"https://github.com/facebookresearch/memory/blob/HEAD/lingua/distributed.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d09c7072d82024f4"}},{"code_sha256_prefix":"faca6fc3f7126a93","entry":"distribute_data_to_rank","repo":"facebookresearch/memory","repo_kind":"official","path":"lingua/data.py","file_url":"https://github.com/facebookresearch/memory/blob/HEAD/lingua/data.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"faca6fc3f7126a93"}},{"code_sha256_prefix":"3c8fff70712ff33a","entry":"find_and_sanitize_chunks","repo":"facebookresearch/memory","repo_kind":"official","path":"lingua/data.py","file_url":"https://github.com/facebookresearch/memory/blob/HEAD/lingua/data.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"3c8fff70712ff33a"}},{"code_sha256_prefix":"eae71867f83f5695","entry":"get_device_mesh","repo":"facebookresearch/memory","repo_kind":"official","path":"lingua/distributed.py","file_url":"https://github.com/facebookresearch/memory/blob/HEAD/lingua/distributed.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"eae71867f83f5695"}},{"code_sha256_prefix":"81d8582d1fab7bda","entry":"get_num_params","repo":"facebookresearch/memory","repo_kind":"official","path":"lingua/metrics.py","file_url":"https://github.com/facebookresearch/memory/blob/HEAD/lingua/metrics.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"81d8582d1fab7bda"}},{"code_sha256_prefix":"22fbaa2caf605450","entry":"log_stats","repo":"facebookresearch/memory","repo_kind":"official","path":"lingua/probe.py","file_url":"https://github.com/facebookresearch/memory/blob/HEAD/lingua/probe.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"22fbaa2caf605450"}},{"code_sha256_prefix":"758fe7262f8fa41e","entry":"lr_cosine","repo":"facebookresearch/memory","repo_kind":"official","path":"lingua/optim.py","file_url":"https://github.com/facebookresearch/memory/blob/HEAD/lingua/optim.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"758fe7262f8fa41e"}},{"code_sha256_prefix":"90403d20388a5636","entry":"lr_inv_sqrt","repo":"facebookresearch/memory","repo_kind":"official","path":"lingua/optim.py","file_url":"https://github.com/facebookresearch/memory/blob/HEAD/lingua/optim.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"90403d20388a5636"}},{"code_sha256_prefix":"4e2454a79a8338d0","entry":"lr_linear","repo":"facebookresearch/memory","repo_kind":"official","path":"lingua/optim.py","file_url":"https://github.com/facebookresearch/memory/blob/HEAD/lingua/optim.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4e2454a79a8338d0"}},{"code_sha256_prefix":"30e68c9cadfddd32","entry":"matmul","repo":"facebookresearch/memory","repo_kind":"official","path":"lingua/float8.py","file_url":"https://github.com/facebookresearch/memory/blob/HEAD/lingua/float8.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"30e68c9cadfddd32"}},{"code_sha256_prefix":"aaa4b37e564944fe","entry":"named_replace","repo":"facebookresearch/memory","repo_kind":"official","path":"lingua/float8.py","file_url":"https://github.com/facebookresearch/memory/blob/HEAD/lingua/float8.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"aaa4b37e564944fe"}},{"code_sha256_prefix":"2848341791f9a450","entry":"scale","repo":"facebookresearch/memory","repo_kind":"official","path":"lingua/float8.py","file_url":"https://github.com/facebookresearch/memory/blob/HEAD/lingua/float8.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2848341791f9a450"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}