{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/adaptive-attention-span-in-transformers","title":"Adaptive Attention Span in Transformers","arxiv_id":"1905.07799","date":"2019-05-19","proceeding":"ACL 2019 7","authors":["Sainbayar Sukhbaatar","Edouard Grave","Piotr Bojanowski","Armand Joulin"],"abstract":"We propose a novel self-attention mechanism that can learn its optimal attention span. This allows us to extend significantly the maximum context size used in Transformer, while maintaining control over their memory footprint and computational time. We show the effectiveness of our approach on the task of character level language modeling, where we achieve state-of-the-art performances on text8 and enwiki8 by using a maximum context of 8k characters.","url_abs":"https://arxiv.org/abs/1905.07799v2","url_pdf":"https://arxiv.org/pdf/1905.07799v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"adaptive-attention-span-in-transformers","repo_url":"https://github.com/facebookresearch/adaptive-span","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"adaptive-attention-span-in-transformers","repo_url":"https://github.com/JoeRoussy/adaptive-attention-in-cv","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"adaptive-attention-span-in-transformers","repo_url":"https://github.com/jerrodparker20/adaptive-transformers-in-rl","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"adaptive-attention-span-in-transformers","repo_url":"https://github.com/lancopku/Explicit-Sparse-Transformer","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"adaptive-attention-span-in-transformers","repo_url":"https://github.com/ofirpress/sandwich_transformer","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"adaptive-attention-span-in-transformers","repo_url":"https://github.com/prajjwal1/adaptive_transformer","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"adaptive-attention-span-in-transformers","repo_url":"https://github.com/prajjwal1/fluence","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"adaptive-attention-span-in-transformers","repo_url":"https://github.com/pwc-1/Paper-9/tree/main/7/Knowing-When-to-Look-Adaptive-Attention","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null}],"tasks":[{"task_slug":null,"task_name":"8k"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"adaptive-masking","method_name":"Adaptive Masking"},{"method_slug":"adaptive-span-transformer","method_name":"Adaptive Span Transformer"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"embedding-dropout","method_name":"Embedding Dropout"},{"method_slug":"l1-regularization","method_name":"L1 Regularization"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"relu","method_name":"ReLU"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/language-modelling-on-text8","task":"Language Modelling","dataset":"Text8","model":"24L Transformer + 8K adaptive span","rank_in_archive_order":4,"of":24,"metrics":{"Bit per Character (BPC)":"1.07","Number of params":"209M"},"uses_additional_data":false},{"leaderboard":"/sota/language-modelling-on-text8","task":"Language Modelling","dataset":"Text8","model":"12L Transformer + 8K adaptive span","rank_in_archive_order":8,"of":24,"metrics":{"Bit per Character (BPC)":"1.11","Number of params":"38M"},"uses_additional_data":false},{"leaderboard":"/sota/language-modelling-on-enwiki8","task":"Language Modelling","dataset":"enwik8","model":"Transformer (24 layers, 8k adaptive span)","rank_in_archive_order":11,"of":42,"metrics":{"Bit per Character (BPC)":"0.98","Number of params":"209M"},"uses_additional_data":false},{"leaderboard":"/sota/language-modelling-on-enwiki8","task":"Language Modelling","dataset":"enwik8","model":"Transformer (12 layers, 8k adaptive span)","rank_in_archive_order":20,"of":42,"metrics":{"Bit per Character (BPC)":"1.02","Number of params":"39M"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1905.07799","mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}