{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/augmenting-self-attention-with-persistent","title":"Augmenting Self-attention with Persistent Memory","arxiv_id":"1907.01470","date":"2019-07-02","proceeding":null,"authors":["Sainbayar Sukhbaatar","Edouard Grave","Guillaume Lample","Herve Jegou","Armand Joulin"],"abstract":"Transformer networks have lead to important progress in language modeling and machine translation. These models include two consecutive modules, a feed-forward layer and a self-attention layer. The latter allows the network to capture long term dependencies and are often regarded as the key ingredient in the success of Transformers. Building upon this intuition, we propose a new model that solely consists of attention layers. More precisely, we augment the self-attention layers with persistent memory vectors that play a similar role as the feed-forward layer. Thanks to these vectors, we can remove the feed-forward layer without degrading the performance of a transformer. Our evaluation shows the benefits brought by our model on standard character and word level language modeling benchmarks.","url_abs":"https://arxiv.org/abs/1907.01470v1","url_pdf":"https://arxiv.org/pdf/1907.01470v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"augmenting-self-attention-with-persistent","repo_url":"https://github.com/facebookresearch/adaptive-span","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"augmenting-self-attention-with-persistent","repo_url":"https://github.com/lucidrains/x-transformers","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"translation","task_name":"Translation"}],"methods":[{"method_slug":"adagrad","method_name":"AdaGrad"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"adaptive-masking","method_name":"Adaptive Masking"},{"method_slug":"all-attention-layer","method_name":"All-Attention Layer"},{"method_slug":"l1-regularization","method_name":"L1 Regularization"}],"datasets_introduced":[],"methods_introduced":[{"slug":"all-attention-layer","name":"All-Attention Layer","full_name":"All-Attention Layer"}],"results":[{"leaderboard":"/sota/language-modelling-on-text8","task":"Language Modelling","dataset":"Text8","model":"All-attention network - 36 layers","rank_in_archive_order":6,"of":24,"metrics":{"Bit per Character (BPC)":"1.08","Number of params":"114M"},"uses_additional_data":false},{"leaderboard":"/sota/language-modelling-on-text8","task":"Language Modelling","dataset":"Text8","model":"All-attention network - 18 layers","rank_in_archive_order":9,"of":24,"metrics":{"Bit per Character (BPC)":"1.11","Number of params":"38M"},"uses_additional_data":false},{"leaderboard":"/sota/language-modelling-on-wikitext-103","task":"Language Modelling","dataset":"WikiText-103","model":"All-attention network (36 layers)","rank_in_archive_order":44,"of":89,"metrics":{"Number of params":"133M","Test perplexity":"20.6","Validation perplexity":"19.7"},"uses_additional_data":false},{"leaderboard":"/sota/language-modelling-on-enwiki8","task":"Language Modelling","dataset":"enwik8","model":"All-attention network (18 layers)","rank_in_archive_order":19,"of":42,"metrics":{"Bit per Character (BPC)":"1.01","Number of params":"39M"},"uses_additional_data":false},{"leaderboard":"/sota/language-modelling-on-enwiki8","task":"Language Modelling","dataset":"enwik8","model":"All-attention network (36 layers)","rank_in_archive_order":42,"of":42,"metrics":{"Number of params":"114M"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1907.01470","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1907.01470"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/adaptive-span","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/lucidrains/x-transformers","reach":null}],"summary":{"ran_honours":2,"ran_violates":3},"by_repo_kind":{"listed":{"samples":1,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":4,"samples":[{"code_sha256_prefix":"7f1040f5e3991d5e","entry":"identity","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_honours","verification_level":2,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"7f1040f5e3991d5e"}},{"code_sha256_prefix":"4a06bf1ac86f840a","entry":"compact","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"4a06bf1ac86f840a"}},{"code_sha256_prefix":"60fff7c3c400d7ff","entry":"default","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"60fff7c3c400d7ff"}},{"code_sha256_prefix":"aa5486a3650902d8","entry":"exists","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"aa5486a3650902d8"}},{"code_sha256_prefix":"123a841a20bb0ebd","entry":"first","repo":"lucidrains/x-transformers","repo_kind":"listed","path":"x_transformers/x_transformers.py","file_url":"https://github.com/lucidrains/x-transformers/blob/HEAD/x_transformers/x_transformers.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"123a841a20bb0ebd"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}