{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/mega-moving-average-equipped-gated-attention","title":"Mega: Moving Average Equipped Gated Attention","arxiv_id":"2209.10655","date":"2022-09-21","proceeding":null,"authors":["Xuezhe Ma","Chunting Zhou","Xiang Kong","Junxian He","Liangke Gui","Graham Neubig","Jonathan May","Luke Zettlemoyer"],"abstract":"The design choices in the Transformer attention mechanism, including weak inductive bias and quadratic computational complexity, have limited its application for modeling long sequences. In this paper, we introduce Mega, a simple, theoretically grounded, single-head gated attention mechanism equipped with (exponential) moving average to incorporate inductive bias of position-aware local dependencies into the position-agnostic attention mechanism. We further propose a variant of Mega that offers linear time and space complexity yet yields only minimal quality loss, by efficiently splitting the whole sequence into multiple chunks with fixed length. Extensive experiments on a wide range of sequence modeling benchmarks, including the Long Range Arena, neural machine translation, auto-regressive language modeling, and image and speech classification, show that Mega achieves significant improvements over other sequence models, including variants of Transformers and recent state space models.","url_abs":"https://arxiv.org/abs/2209.10655v3","url_pdf":"https://arxiv.org/pdf/2209.10655v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"mega-moving-average-equipped-gated-attention","repo_url":"https://github.com/facebookresearch/mega","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"mega-moving-average-equipped-gated-attention","repo_url":"https://github.com/ZIZUN/MAFiD","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"mega-moving-average-equipped-gated-attention","repo_url":"https://github.com/ethanbar11/ssm_2d","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"mega-moving-average-equipped-gated-attention","repo_url":"https://github.com/huggingface/transformers","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"mega-moving-average-equipped-gated-attention","repo_url":"https://github.com/linghao-jin/canmt-challenges","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"mega-moving-average-equipped-gated-attention","repo_url":"https://github.com/lucidrains/gated-state-spaces-pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"mega-moving-average-equipped-gated-attention","repo_url":"https://github.com/pwc-1/Paper-9/tree/main/2/mega","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null}],"tasks":[{"task_slug":"image-classification","task_name":"Image Classification"},{"task_slug":"inductive-bias","task_name":"Inductive Bias"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"listops","task_name":"ListOps"},{"task_slug":"long-range-modeling","task_name":"Long-range modeling"},{"task_slug":"machine-translation","task_name":"Machine Translation"},{"task_slug":null,"task_name":"Position"},{"task_slug":"state-space-models","task_name":"State Space Models"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/image-classification-on-imagenet","task":"Image Classification","dataset":"ImageNet","model":"Mega","rank_in_archive_order":543,"of":1060,"metrics":{"Number of params":"90M","Top 1 Accuracy":"82.4%"},"uses_additional_data":false},{"leaderboard":"/sota/language-modelling-on-wikitext-103","task":"Language Modelling","dataset":"WikiText-103","model":"Mega","rank_in_archive_order":30,"of":89,"metrics":{"Number of params":"252M","Test perplexity":"18.07"},"uses_additional_data":false},{"leaderboard":"/sota/machine-translation-on-wmt2014-english-german","task":"Machine Translation","dataset":"WMT2014 English-German","model":"Mega","rank_in_archive_order":33,"of":91,"metrics":{"BLEU score":"29.01","Number of Params":"67M","SacreBLEU":"27.96"},"uses_additional_data":false},{"leaderboard":"/sota/machine-translation-on-wmt2014-german-english","task":"Machine Translation","dataset":"WMT2014 German-English","model":"Mega","rank_in_archive_order":4,"of":16,"metrics":{"BLEU score":"33.12"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2209.10655","atlas_url":"https://app.syntology.ai/?focus=2209.10655","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2209.10655"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/huggingface/transformers","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/mega","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ethanbar11/ssm_2d","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/lucidrains/gated-state-spaces-pytorch","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/pwc-1/Paper-9/tree/main/2/mega","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/linghao-jin/canmt-challenges","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ZIZUN/MAFiD","reach":null}],"summary":{"ran":8,"ran_draft_wrong":2,"ran_fixture":2,"unverified":1},"by_repo_kind":{"official":{"samples":3,"ran":3,"repositories":1},"listed":{"samples":10,"ran":9,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":8,"samples":[{"code_sha256_prefix":"7cf1fc5a8fb4711f","entry":"DSS","repo":"lucidrains/gated-state-spaces-pytorch","repo_kind":"listed","path":"gated_state_spaces_pytorch/gss.py","file_url":"https://github.com/lucidrains/gated-state-spaces-pytorch/blob/HEAD/gated_state_spaces_pytorch/gss.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7cf1fc5a8fb4711f"}},{"code_sha256_prefix":"96cbb5e9ca5b6be0","entry":"Embedding","repo":"facebookresearch/mega","repo_kind":"official","path":"fairseq/models/mega.py","file_url":"https://github.com/facebookresearch/mega/blob/HEAD/fairseq/models/mega.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"96cbb5e9ca5b6be0"}},{"code_sha256_prefix":"e0fb8682bea0936e","entry":"GSS","repo":"lucidrains/gated-state-spaces-pytorch","repo_kind":"listed","path":"gated_state_spaces_pytorch/gss.py","file_url":"https://github.com/lucidrains/gated-state-spaces-pytorch/blob/HEAD/gated_state_spaces_pytorch/gss.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e0fb8682bea0936e"}},{"code_sha256_prefix":"28048afa2b58e989","entry":"LaplacianAttnFn","repo":"ZIZUN/MAFiD","repo_kind":"listed","path":"src/model.py","file_url":"https://github.com/ZIZUN/MAFiD/blob/HEAD/src/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"28048afa2b58e989"}},{"code_sha256_prefix":"8cd8cb0d1e9e63e4","entry":"Linear","repo":"facebookresearch/mega","repo_kind":"official","path":"fairseq/models/mega.py","file_url":"https://github.com/facebookresearch/mega/blob/HEAD/fairseq/models/mega.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8cd8cb0d1e9e63e4"}},{"code_sha256_prefix":"d65478e4afde1394","entry":"MultiHeadedEMA","repo":"ZIZUN/MAFiD","repo_kind":"listed","path":"src/model.py","file_url":"https://github.com/ZIZUN/MAFiD/blob/HEAD/src/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d65478e4afde1394"}},{"code_sha256_prefix":"fd172a441158b8cf","entry":"OffsetScale","repo":"ZIZUN/MAFiD","repo_kind":"listed","path":"src/model.py","file_url":"https://github.com/ZIZUN/MAFiD/blob/HEAD/src/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fd172a441158b8cf"}},{"code_sha256_prefix":"bf0381787b2cfd85","entry":"SingleHeadedAttention","repo":"ZIZUN/MAFiD","repo_kind":"listed","path":"src/model.py","file_url":"https://github.com/ZIZUN/MAFiD/blob/HEAD/src/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bf0381787b2cfd85"}},{"code_sha256_prefix":"75a6e13f59e67f7e","entry":"T5RelativePositionBias","repo":"ZIZUN/MAFiD","repo_kind":"listed","path":"src/model.py","file_url":"https://github.com/ZIZUN/MAFiD/blob/HEAD/src/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"75a6e13f59e67f7e"}},{"code_sha256_prefix":"af830e39fdd08afa","entry":"append_dims","repo":"ZIZUN/MAFiD","repo_kind":"listed","path":"src/model.py","file_url":"https://github.com/ZIZUN/MAFiD/blob/HEAD/src/model.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"af830e39fdd08afa"}},{"code_sha256_prefix":"c3e96d57908b9ca9","entry":"conv1d_fft","repo":"ZIZUN/MAFiD","repo_kind":"listed","path":"src/model.py","file_url":"https://github.com/ZIZUN/MAFiD/blob/HEAD/src/model.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c3e96d57908b9ca9"}},{"code_sha256_prefix":"82bdf940665aeeb4","entry":"safe_readline","repo":"facebookresearch/mega","repo_kind":"official","path":"fairseq/binarizer.py","file_url":"https://github.com/facebookresearch/mega/blob/HEAD/fairseq/binarizer.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"82bdf940665aeeb4"}},{"code_sha256_prefix":"cfbd8ca4d8912d27","entry":"MegaLayer","repo":"ZIZUN/MAFiD","repo_kind":"listed","path":"src/model.py","file_url":"https://github.com/ZIZUN/MAFiD/blob/HEAD/src/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"cfbd8ca4d8912d27"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}