{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/multi-branch-attentive-transformer","title":"Multi-branch Attentive Transformer","arxiv_id":"2006.10270","date":"2020-06-18","proceeding":null,"authors":["Yang Fan","Shufang Xie","Yingce Xia","Lijun Wu","Tao Qin","Xiang-Yang Li","Tie-Yan Liu"],"abstract":"While the multi-branch architecture is one of the key ingredients to the success of computer vision tasks, it has not been well investigated in natural language processing, especially sequence learning tasks. In this work, we propose a simple yet effective variant of Transformer called multi-branch attentive Transformer (briefly, MAT), where the attention layer is the average of multiple branches and each branch is an independent multi-head attention layer. We leverage two training techniques to regularize the training: drop-branch, which randomly drops individual branches during training, and proximal initialization, which uses a pre-trained Transformer model to initialize multiple branches. Experiments on machine translation, code generation and natural language understanding demonstrate that such a simple variant of Transformer brings significant improvements. Our code is available at \\url{https://github.com/HA-Transformer}.","url_abs":"https://arxiv.org/abs/2006.10270v2","url_pdf":"https://arxiv.org/pdf/2006.10270v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"multi-branch-attentive-transformer","repo_url":"https://github.com/HA-Transformer/HA-Transformer","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"code-generation","task_name":"Code Generation"},{"task_slug":"machine-translation","task_name":"Machine Translation"},{"task_slug":"natural-language-understanding","task_name":"Natural Language Understanding"},{"task_slug":"translation","task_name":"Translation"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"relu","method_name":"ReLU"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/machine-translation-on-iwslt2014-german","task":"Machine Translation","dataset":"IWSLT2014 German-English","model":"MAT","rank_in_archive_order":18,"of":34,"metrics":{"BLEU score":"36.22"},"uses_additional_data":false},{"leaderboard":"/sota/machine-translation-on-wmt2014-english-german","task":"Machine Translation","dataset":"WMT2014 English-German","model":"MAT","rank_in_archive_order":91,"of":91,"metrics":{"SacreBLEU":"29.9"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2006.10270","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2006.10270"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/HA-Transformer/HA-Transformer","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":2,"unverified":1},"by_repo_kind":{"listed":{"samples":3,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":3,"samples":[{"code_sha256_prefix":"96cbb5e9ca5b6be0","entry":"Embedding","repo":"HA-Transformer/HA-Transformer","repo_kind":"listed","path":"fairseq/models/transformer.py","file_url":"https://github.com/HA-Transformer/HA-Transformer/blob/HEAD/fairseq/models/transformer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"96cbb5e9ca5b6be0"}},{"code_sha256_prefix":"8cd8cb0d1e9e63e4","entry":"Linear","repo":"HA-Transformer/HA-Transformer","repo_kind":"listed","path":"fairseq/models/transformer.py","file_url":"https://github.com/HA-Transformer/HA-Transformer/blob/HEAD/fairseq/models/transformer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"8cd8cb0d1e9e63e4"}},{"code_sha256_prefix":"f650c3fc2ff8f602","entry":"merge_branches","repo":"HA-Transformer/HA-Transformer","repo_kind":"listed","path":"fairseq/models/transformer_mb/transformer_mb_layer.py","file_url":"https://github.com/HA-Transformer/HA-Transformer/blob/HEAD/fairseq/models/transformer_mb/transformer_mb_layer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"f650c3fc2ff8f602"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}