{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/fnet-mixing-tokens-with-fourier-transforms","title":"FNet: Mixing Tokens with Fourier Transforms","arxiv_id":"2105.03824","date":"2021-05-09","proceeding":"NAACL 2022 7","authors":["James Lee-Thorp","Joshua Ainslie","Ilya Eckstein","Santiago Ontanon"],"abstract":"We show that Transformer encoder architectures can be sped up, with limited accuracy costs, by replacing the self-attention sublayers with simple linear transformations that \"mix\" input tokens. These linear mixers, along with standard nonlinearities in feed-forward layers, prove competent at modeling semantic relationships in several text classification tasks. Most surprisingly, we find that replacing the self-attention sublayer in a Transformer encoder with a standard, unparameterized Fourier Transform achieves 92-97% of the accuracy of BERT counterparts on the GLUE benchmark, but trains 80% faster on GPUs and 70% faster on TPUs at standard 512 input lengths. At longer input lengths, our FNet model is significantly faster: when compared to the \"efficient\" Transformers on the Long Range Arena benchmark, FNet matches the accuracy of the most accurate models, while outpacing the fastest models across all sequence lengths on GPUs (and across relatively shorter lengths on TPUs). Finally, FNet has a light memory footprint and is particularly efficient at smaller model sizes; for a fixed speed and accuracy budget, small FNet models outperform Transformer counterparts.","url_abs":"https://arxiv.org/abs/2105.03824v4","url_pdf":"https://arxiv.org/pdf/2105.03824v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"fnet-mixing-tokens-with-fourier-transforms","repo_url":"https://github.com/google-research/google-research","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"tf","reach":null},{"paper_slug":"fnet-mixing-tokens-with-fourier-transforms","repo_url":"https://github.com/amoramine/FNet_classification","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"fnet-mixing-tokens-with-fourier-transforms","repo_url":"https://github.com/amoramine/FNet_with_BART_classification","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"fnet-mixing-tokens-with-fourier-transforms","repo_url":"https://github.com/erksch/fnet-pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"fnet-mixing-tokens-with-fourier-transforms","repo_url":"https://github.com/facebookresearch/xformers","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"fnet-mixing-tokens-with-fourier-transforms","repo_url":"https://github.com/jaketae/fnet","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"fnet-mixing-tokens-with-fourier-transforms","repo_url":"https://github.com/vineet54/fnet-google-pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"fnet-mixing-tokens-with-fourier-transforms","repo_url":"https://github.com/2024-MindSpore-1/Code2/tree/main/model-1/fnet","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null},{"paper_slug":"fnet-mixing-tokens-with-fourier-transforms","repo_url":"https://github.com/HJHGJGHHG/Paddle-FNet","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"paddle","reach":{"status":"ok"}},{"paper_slug":"fnet-mixing-tokens-with-fourier-transforms","repo_url":"https://github.com/abdelghanibelgaid/FNet-TensorFlow-PyTorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"tf","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"fnet-mixing-tokens-with-fourier-transforms","repo_url":"https://github.com/labmlai/annotated_deep_learning_paper_implementations","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":null},{"paper_slug":"fnet-mixing-tokens-with-fourier-transforms","repo_url":"https://github.com/yangyucheng000/University/tree/main/model-2/fnet","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null}],"tasks":[{"task_slug":"linguistic-acceptability","task_name":"Linguistic Acceptability"},{"task_slug":"machine-translation","task_name":"Machine Translation"},{"task_slug":"natural-language-inference","task_name":"Natural Language Inference"},{"task_slug":"paraphrase-identification","task_name":"Paraphrase Identification"},{"task_slug":"semantic-textual-similarity","task_name":"Semantic Textual Similarity"},{"task_slug":"sentiment-analysis","task_name":"Sentiment Analysis"},{"task_slug":"text-classification","task_name":"Text Classification"},{"task_slug":"transfer-learning","task_name":"Transfer Learning"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/linguistic-acceptability-on-cola","task":"Linguistic Acceptability","dataset":"CoLA","model":"FNet-Large","rank_in_archive_order":9,"of":43,"metrics":{"Accuracy":"78%"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-multinli","task":"Natural Language Inference","dataset":"MultiNLI","model":"BERT-Large","rank_in_archive_order":17,"of":67,"metrics":{"Matched":"88","Mismatched":"88"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-multinli","task":"Natural Language Inference","dataset":"MultiNLI","model":"FNet-Large","rank_in_archive_order":46,"of":67,"metrics":{"Matched":"78","Mismatched":"76"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-qnli","task":"Natural Language Inference","dataset":"QNLI","model":"FNet-Large","rank_in_archive_order":40,"of":43,"metrics":{"Accuracy":"85%"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-rte","task":"Natural Language Inference","dataset":"RTE","model":"FNet-Large","rank_in_archive_order":57,"of":90,"metrics":{"Accuracy":"69%"},"uses_additional_data":false},{"leaderboard":"/sota/paraphrase-identification-on-quora-question","task":"Paraphrase Identification","dataset":"Quora Question Pairs","model":"FNet-Large","rank_in_archive_order":5,"of":31,"metrics":{"F1":"85"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-mrpc","task":"Semantic Textual Similarity","dataset":"MRPC","model":"FNet-Large","rank_in_archive_order":23,"of":45,"metrics":{"Accuracy":"88%"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-sts-benchmark","task":"Semantic Textual Similarity","dataset":"STS Benchmark","model":"FNet-Large","rank_in_archive_order":50,"of":66,"metrics":{"Spearman Correlation":"0.84"},"uses_additional_data":false},{"leaderboard":"/sota/sentiment-analysis-on-sst-2-binary","task":"Sentiment Analysis","dataset":"SST-2 Binary classification","model":"FNet-Large","rank_in_archive_order":38,"of":87,"metrics":{"Accuracy":"94"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2105.03824","atlas_url":"https://app.syntology.ai/?focus=2105.03824","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.03824"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/google-research/google-research","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/erksch/fnet-pytorch","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/2024-MindSpore-1/Code2/tree/main/model-1/fnet","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jaketae/fnet","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/HJHGJGHHG/Paddle-FNet","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/amoramine/FNet_with_BART_classification","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/amoramine/FNet_classification","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yangyucheng000/University/tree/main/model-2/fnet","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/labmlai/annotated_deep_learning_paper_implementations","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/vineet54/fnet-google-pytorch","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/abdelghanibelgaid/FNet-TensorFlow-PyTorch","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/xformers","reach":null}],"summary":{"ran_honours":1,"ran_draft_wrong":1},"by_repo_kind":{"listed":{"samples":2,"ran":2,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":1,"samples":[{"code_sha256_prefix":"c92b7a737063ef80","entry":"evaluate","repo":"amoramine/FNet_with_BART_classification","repo_kind":"listed","path":"fnet.py","file_url":"https://github.com/amoramine/FNet_with_BART_classification/blob/HEAD/fnet.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c92b7a737063ef80"}},{"code_sha256_prefix":"8b5fcaaabe030716","entry":"fourier_transform","repo":"jaketae/fnet","repo_kind":"listed","path":"fnet/core.py","file_url":"https://github.com/jaketae/fnet/blob/HEAD/fnet/core.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8b5fcaaabe030716"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}