{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/mixture-of-attention-heads-selecting","title":"Mixture of Attention Heads: Selecting Attention Heads Per Token","arxiv_id":"2210.05144","date":"2022-10-11","proceeding":null,"authors":["Xiaofeng Zhang","Yikang Shen","Zeyu Huang","Jie zhou","Wenge Rong","Zhang Xiong"],"abstract":"Mixture-of-Experts (MoE) networks have been proposed as an efficient way to scale up model capacity and implement conditional computing. However, the study of MoE components mostly focused on the feedforward layer in Transformer architecture. This paper proposes the Mixture of Attention Heads (MoA), a new architecture that combines multi-head attention with the MoE mechanism. MoA includes a set of attention heads that each has its own set of parameters. Given an input, a router dynamically selects a subset of $k$ attention heads per token. This conditional computation schema allows MoA to achieve stronger performance than the standard multi-head attention layer. Furthermore, the sparsely gated MoA can easily scale up the number of attention heads and the number of parameters while preserving computational efficiency. In addition to the performance improvements, MoA also automatically differentiates heads' utilities, providing a new perspective to discuss the model's interpretability. We conducted experiments on several important tasks, including Machine Translation and Masked Language Modeling. Experiments have shown promising results on several tasks against strong baselines that involve large and very deep models.","url_abs":"https://arxiv.org/abs/2210.05144v1","url_pdf":"https://arxiv.org/pdf/2210.05144v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"mixture-of-attention-heads-selecting","repo_url":"https://github.com/yikangshen/moa","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"BSD-3-Clause"}},{"paper_slug":"mixture-of-attention-heads-selecting","repo_url":"https://github.com/yikangshen/megablocks","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"computational-efficiency","task_name":"Computational Efficiency"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"machine-translation","task_name":"Machine Translation"},{"task_slug":"masked-language-modeling","task_name":"Masked Language Modeling"},{"task_slug":"mixture-of-experts","task_name":"Mixture-of-Experts"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2210.05144","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.05144"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/sovrasov/flops-counter.pytorch","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yikangshen/megablocks","reach":null},{"provenance":"deterministic:regex_extraction","url":"https://github.com/tensorflow/tensor2tensor","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yikangshen/moa","reach":{"status":"ok","spdx":"BSD-3-Clause"}}],"summary":{"ran_draft_wrong":3,"ran":2,"ran_honours":1,"unverified":11},"by_repo_kind":{"official":{"samples":5,"ran":5,"repositories":1},"listed":{"samples":1,"ran":1,"repositories":1},"found_in_text":{"samples":11,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"96cbb5e9ca5b6be0","entry":"Embedding","repo":"yikangshen/moa","repo_kind":"official","path":"transformer_base.py","file_url":"https://github.com/yikangshen/moa/blob/HEAD/transformer_base.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"96cbb5e9ca5b6be0"}},{"code_sha256_prefix":"8cd8cb0d1e9e63e4","entry":"Linear","repo":"yikangshen/moa","repo_kind":"official","path":"transformer_decoder.py","file_url":"https://github.com/yikangshen/moa/blob/HEAD/transformer_decoder.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"8cd8cb0d1e9e63e4"}},{"code_sha256_prefix":"96b6713ff1c322ca","entry":"label_smoothed_nll_loss","repo":"yikangshen/moa","repo_kind":"official","path":"moa_loss.py","file_url":"https://github.com/yikangshen/moa/blob/HEAD/moa_loss.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"96b6713ff1c322ca"}},{"code_sha256_prefix":"c3fcda996a38b833","entry":"module_name_fordropout","repo":"yikangshen/moa","repo_kind":"official","path":"transformer_encoder.py","file_url":"https://github.com/yikangshen/moa/blob/HEAD/transformer_encoder.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"c3fcda996a38b833"}},{"code_sha256_prefix":"e63f8c52be2a8547","entry":"module_name_fordropout","repo":"yikangshen/moa","repo_kind":"official","path":"transformer_decoder.py","file_url":"https://github.com/yikangshen/moa/blob/HEAD/transformer_decoder.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"e63f8c52be2a8547"}},{"code_sha256_prefix":"7630a3eb1f9be798","entry":"promote_scalar","repo":"yikangshen/megablocks","repo_kind":"listed","path":"megablocks/layers/moa.py","file_url":"https://github.com/yikangshen/megablocks/blob/HEAD/megablocks/layers/moa.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7630a3eb1f9be798"}},{"code_sha256_prefix":"17cd6272d876a166","entry":"accumulate_flops","repo":"sovrasov/flops-counter.pytorch","repo_kind":"found_in_text","path":"ptflops/pytorch_engine.py","file_url":"https://github.com/sovrasov/flops-counter.pytorch/blob/HEAD/ptflops/pytorch_engine.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"17cd6272d876a166"}},{"code_sha256_prefix":"81acb062e01cd209","entry":"addmm_flop","repo":"sovrasov/flops-counter.pytorch","repo_kind":"found_in_text","path":"ptflops/aten_ops.py","file_url":"https://github.com/sovrasov/flops-counter.pytorch/blob/HEAD/ptflops/aten_ops.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"81acb062e01cd209"}},{"code_sha256_prefix":"dcd80b80811a2b5b","entry":"bert_input_constructor","repo":"sovrasov/flops-counter.pytorch","repo_kind":"found_in_text","path":"samples/bert.py","file_url":"https://github.com/sovrasov/flops-counter.pytorch/blob/HEAD/samples/bert.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dcd80b80811a2b5b"}},{"code_sha256_prefix":"10741db0a4b99b7e","entry":"flops_to_string","repo":"sovrasov/flops-counter.pytorch","repo_kind":"found_in_text","path":"ptflops/utils.py","file_url":"https://github.com/sovrasov/flops-counter.pytorch/blob/HEAD/ptflops/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"10741db0a4b99b7e"}},{"code_sha256_prefix":"46f8ed51e754d72a","entry":"get_flops_pytorch","repo":"sovrasov/flops-counter.pytorch","repo_kind":"found_in_text","path":"ptflops/pytorch_engine.py","file_url":"https://github.com/sovrasov/flops-counter.pytorch/blob/HEAD/ptflops/pytorch_engine.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"46f8ed51e754d72a"}},{"code_sha256_prefix":"46e7df6447a3d499","entry":"get_model_complexity_info","repo":"sovrasov/flops-counter.pytorch","repo_kind":"found_in_text","path":"ptflops/flops_counter.py","file_url":"https://github.com/sovrasov/flops-counter.pytorch/blob/HEAD/ptflops/flops_counter.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"46e7df6447a3d499"}},{"code_sha256_prefix":"55836471a41bf03e","entry":"matmul_flop","repo":"sovrasov/flops-counter.pytorch","repo_kind":"found_in_text","path":"ptflops/aten_ops.py","file_url":"https://github.com/sovrasov/flops-counter.pytorch/blob/HEAD/ptflops/aten_ops.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"55836471a41bf03e"}},{"code_sha256_prefix":"492162380895ff7c","entry":"params_to_string","repo":"sovrasov/flops-counter.pytorch","repo_kind":"found_in_text","path":"ptflops/utils.py","file_url":"https://github.com/sovrasov/flops-counter.pytorch/blob/HEAD/ptflops/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"492162380895ff7c"}},{"code_sha256_prefix":"de6b5f2a68a9e570","entry":"print_model_with_flops","repo":"sovrasov/flops-counter.pytorch","repo_kind":"found_in_text","path":"ptflops/pytorch_engine.py","file_url":"https://github.com/sovrasov/flops-counter.pytorch/blob/HEAD/ptflops/pytorch_engine.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"de6b5f2a68a9e570"}},{"code_sha256_prefix":"185b3f3ea726676d","entry":"prod","repo":"sovrasov/flops-counter.pytorch","repo_kind":"found_in_text","path":"ptflops/aten_ops.py","file_url":"https://github.com/sovrasov/flops-counter.pytorch/blob/HEAD/ptflops/aten_ops.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"185b3f3ea726676d"}},{"code_sha256_prefix":"4883be8641ea59fd","entry":"rnn_flops","repo":"sovrasov/flops-counter.pytorch","repo_kind":"found_in_text","path":"ptflops/pytorch_ops.py","file_url":"https://github.com/sovrasov/flops-counter.pytorch/blob/HEAD/ptflops/pytorch_ops.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4883be8641ea59fd"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}