{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/an-attention-free-transformer-1","title":"An Attention Free Transformer","arxiv_id":"2105.14103","date":"2021-05-28","proceeding":null,"authors":["Shuangfei Zhai","Walter Talbott","Nitish Srivastava","Chen Huang","Hanlin Goh","Ruixiang Zhang","Josh Susskind"],"abstract":"We introduce Attention Free Transformer (AFT), an efficient variant of Transformers that eliminates the need for dot product self attention. In an AFT layer, the key and value are first combined with a set of learned position biases, the result of which is multiplied with the query in an element-wise fashion. This new operation has a memory complexity linear w.r.t. both the context size and the dimension of features, making it compatible to both large input and model sizes. We also introduce AFT-local and AFT-conv, two model variants that take advantage of the idea of locality and spatial weight sharing while maintaining global connectivity. We conduct extensive experiments on two autoregressive modeling tasks (CIFAR10 and Enwik8) as well as an image recognition task (ImageNet-1K classification). We show that AFT demonstrates competitive performance on all the benchmarks, while providing excellent efficiency at the same time.","url_abs":"https://arxiv.org/abs/2105.14103v2","url_pdf":"https://arxiv.org/pdf/2105.14103v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"an-attention-free-transformer-1","repo_url":"https://github.com/ShenDezhou/aft-pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"an-attention-free-transformer-1","repo_url":"https://github.com/hannibal046/nanorwkv","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"an-attention-free-transformer-1","repo_url":"https://github.com/junnyu/rwkv-v2-rnn-paddle","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"paddle","reach":{"status":"ok","spdx":"BSD-2-Clause"}},{"paper_slug":"an-attention-free-transformer-1","repo_url":"https://github.com/lvyufeng/AFT-MindSpore","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"mindspore","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"an-attention-free-transformer-1","repo_url":"https://github.com/nlpodyssey/rwkv","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"BSD-2-Clause"}},{"paper_slug":"an-attention-free-transformer-1","repo_url":"https://github.com/rish-16/aft-pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"an-attention-free-transformer-1","repo_url":"https://github.com/rwkv/rwkv-lm","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"an-attention-free-transformer-1","repo_url":"https://github.com/zarak0/rwkvalgorithm","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"BSD-2-Clause"}},{"paper_slug":"an-attention-free-transformer-1","repo_url":"https://github.com/labmlai/annotated_deep_learning_paper_implementations","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":null},{"paper_slug":"an-attention-free-transformer-1","repo_url":"https://github.com/mindspore-courses/External-Attention-MindSpore/blob/main/model/attention/AFT.py","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null},{"paper_slug":"an-attention-free-transformer-1","repo_url":"https://github.com/pwc-1/Paper-9/tree/main/1/transformer","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null}],"tasks":[{"task_slug":null,"task_name":"Position"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-free-transformer","method_name":"Attention Free Transformer"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[{"slug":"attention-free-transformer","name":"Attention Free Transformer","full_name":"Attention Free Transformer"}],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2105.14103","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.14103"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/rish-16/aft-pytorch","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/hannibal046/nanorwkv","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mindspore-courses/External-Attention-MindSpore/blob/main/model/attention/AFT.py","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/lvyufeng/AFT-MindSpore","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/nlpodyssey/rwkv","reach":{"status":"ok","spdx":"BSD-2-Clause"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ShenDezhou/aft-pytorch","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/labmlai/annotated_deep_learning_paper_implementations","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/pwc-1/Paper-9/tree/main/1/transformer","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/junnyu/rwkv-v2-rnn-paddle","reach":{"status":"ok","spdx":"BSD-2-Clause"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/rwkv/rwkv-lm","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/zarak0/rwkvalgorithm","reach":{"status":"ok","spdx":"BSD-2-Clause"}}],"summary":{"unverified":1},"by_repo_kind":{"listed":{"samples":1,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"24f05fe56a6fc171","entry":"orthogonal","repo":"junnyu/rwkv-v2-rnn-paddle","repo_kind":"listed","path":"src/model.py","file_url":"https://github.com/junnyu/rwkv-v2-rnn-paddle/blob/HEAD/src/model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"24f05fe56a6fc171"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}