{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/incorporating-bert-into-parallel-sequence","title":"Incorporating BERT into Parallel Sequence Decoding with Adapters","arxiv_id":"2010.06138","date":"2020-10-13","proceeding":"NeurIPS 2020 12","authors":["Junliang Guo","Zhirui Zhang","Linli Xu","Hao-Ran Wei","Boxing Chen","Enhong Chen"],"abstract":"While large scale pre-trained language models such as BERT have achieved great success on various natural language understanding tasks, how to efficiently and effectively incorporate them into sequence-to-sequence models and the corresponding text generation tasks remains a non-trivial problem. In this paper, we propose to address this problem by taking two different BERT models as the encoder and decoder respectively, and fine-tuning them by introducing simple and lightweight adapter modules, which are inserted between BERT layers and tuned on the task-specific dataset. In this way, we obtain a flexible and efficient model which is able to jointly leverage the information contained in the source-side and target-side BERT models, while bypassing the catastrophic forgetting problem. Each component in the framework can be considered as a plug-in unit, making the framework flexible and task agnostic. Our framework is based on a parallel sequence decoding algorithm named Mask-Predict considering the bi-directional and conditional independent nature of BERT, and can be adapted to traditional autoregressive decoding easily. We conduct extensive experiments on neural machine translation tasks where the proposed method consistently outperforms autoregressive baselines while reducing the inference latency by half, and achieves $36.49$/$33.57$ BLEU scores on IWSLT14 German-English/WMT14 German-English translation. When adapted to autoregressive decoding, the proposed method achieves $30.60$/$43.56$ BLEU scores on WMT14 English-German/English-French translation, on par with the state-of-the-art baseline models.","url_abs":"https://arxiv.org/abs/2010.06138v1","url_pdf":"https://arxiv.org/pdf/2010.06138v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"incorporating-bert-into-parallel-sequence","repo_url":"https://github.com/lemmonation/abnet","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"machine-translation","task_name":"Machine Translation"},{"task_slug":"natural-language-understanding","task_name":"Natural Language Understanding"},{"task_slug":"text-generation","task_name":"Text Generation"},{"task_slug":"translation","task_name":"Translation"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2010.06138","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.06138"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/lemmonation/abnet","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":3,"ran_honours":1,"ran":1,"unverified":8},"by_repo_kind":{"official":{"samples":13,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"0f786c407fb1ee4c","entry":"swish","repo":"lemmonation/abnet","repo_kind":"official","path":"bert/modeling.py","file_url":"https://github.com/lemmonation/abnet/blob/HEAD/bert/modeling.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":2,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0f786c407fb1ee4c"}},{"code_sha256_prefix":"cf9ffa02a42184af","entry":"whitespace_tokenize","repo":"lemmonation/abnet","repo_kind":"official","path":"bert/tokenization.py","file_url":"https://github.com/lemmonation/abnet/blob/HEAD/bert/tokenization.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":2,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cf9ffa02a42184af"}},{"code_sha256_prefix":"fdc64f4c72036ae4","entry":"gelu","repo":"lemmonation/abnet","repo_kind":"official","path":"bert/modeling.py","file_url":"https://github.com/lemmonation/abnet/blob/HEAD/bert/modeling.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fdc64f4c72036ae4"}},{"code_sha256_prefix":"82bdf940665aeeb4","entry":"safe_readline","repo":"lemmonation/abnet","repo_kind":"official","path":"fairseq/binarizer.py","file_url":"https://github.com/lemmonation/abnet/blob/HEAD/fairseq/binarizer.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"82bdf940665aeeb4"}},{"code_sha256_prefix":"5eff22fa0a651276","entry":"url_to_filename","repo":"lemmonation/abnet","repo_kind":"official","path":"bert/file_utils.py","file_url":"https://github.com/lemmonation/abnet/blob/HEAD/bert/file_utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5eff22fa0a651276"}},{"code_sha256_prefix":"cf9c448c96f48428","entry":"cached_path","repo":"lemmonation/abnet","repo_kind":"official","path":"bert/file_utils.py","file_url":"https://github.com/lemmonation/abnet/blob/HEAD/bert/file_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cf9c448c96f48428"}},{"code_sha256_prefix":"9eda7f80331d658c","entry":"dataset_dest_prefix","repo":"lemmonation/abnet","repo_kind":"official","path":"preprocess.py","file_url":"https://github.com/lemmonation/abnet/blob/HEAD/preprocess.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9eda7f80331d658c"}},{"code_sha256_prefix":"a6aab787fcf8b10d","entry":"filename_to_url","repo":"lemmonation/abnet","repo_kind":"official","path":"bert/file_utils.py","file_url":"https://github.com/lemmonation/abnet/blob/HEAD/bert/file_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a6aab787fcf8b10d"}},{"code_sha256_prefix":"336077aca45a1da4","entry":"load_checkpoint","repo":"lemmonation/abnet","repo_kind":"official","path":"fairseq/checkpoint_utils.py","file_url":"https://github.com/lemmonation/abnet/blob/HEAD/fairseq/checkpoint_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"336077aca45a1da4"}},{"code_sha256_prefix":"c6a6a0734f184953","entry":"load_checkpoint_to_cpu","repo":"lemmonation/abnet","repo_kind":"official","path":"fairseq/checkpoint_utils.py","file_url":"https://github.com/lemmonation/abnet/blob/HEAD/fairseq/checkpoint_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c6a6a0734f184953"}},{"code_sha256_prefix":"b2d82a49dbc70999","entry":"load_model_ensemble","repo":"lemmonation/abnet","repo_kind":"official","path":"fairseq/checkpoint_utils.py","file_url":"https://github.com/lemmonation/abnet/blob/HEAD/fairseq/checkpoint_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b2d82a49dbc70999"}},{"code_sha256_prefix":"b1a7180868dd8d6a","entry":"load_tf_weights_in_bert","repo":"lemmonation/abnet","repo_kind":"official","path":"bert/modeling.py","file_url":"https://github.com/lemmonation/abnet/blob/HEAD/bert/modeling.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b1a7180868dd8d6a"}},{"code_sha256_prefix":"a4e183f24a5ad1a7","entry":"load_vocab","repo":"lemmonation/abnet","repo_kind":"official","path":"bert/tokenization.py","file_url":"https://github.com/lemmonation/abnet/blob/HEAD/bert/tokenization.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a4e183f24a5ad1a7"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}