{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/sequence-level-knowledge-distillation","title":"Sequence-Level Knowledge Distillation","arxiv_id":"1606.07947","date":"2016-06-25","proceeding":"EMNLP 2016 11","authors":["Yoon Kim","Alexander M. Rush"],"abstract":"Neural machine translation (NMT) offers a novel alternative formulation of\ntranslation that is potentially simpler than statistical approaches. However to\nreach competitive performance, NMT models need to be exceedingly large. In this\npaper we consider applying knowledge distillation approaches (Bucila et al.,\n2006; Hinton et al., 2015) that have proven successful for reducing the size of\nneural models in other domains to the problem of NMT. We demonstrate that\nstandard knowledge distillation applied to word-level prediction can be\neffective for NMT, and also introduce two novel sequence-level versions of\nknowledge distillation that further improve performance, and somewhat\nsurprisingly, seem to eliminate the need for beam search (even when applied on\nthe original teacher model). Our best student model runs 10 times faster than\nits state-of-the-art teacher with little loss in performance. It is also\nsignificantly better than a baseline model trained without knowledge\ndistillation: by 4.2/1.7 BLEU with greedy decoding/beam search. Applying weight\npruning on top of knowledge distillation results in a student model that has 13\ntimes fewer parameters than the original teacher model, with a decrease of 0.4\nBLEU.","url_abs":"http://arxiv.org/abs/1606.07947v4","url_pdf":"http://arxiv.org/pdf/1606.07947v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"sequence-level-knowledge-distillation","repo_url":"https://github.com/harvardnlp/nmt-android","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"torch","reach":{"status":"ok","spdx":"GPL-3.0"}},{"paper_slug":"sequence-level-knowledge-distillation","repo_url":"https://github.com/harvardnlp/seq2seq-attn","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"torch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"sequence-level-knowledge-distillation","repo_url":"https://github.com/anonymouscode115/autoregressive_inference","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":null},{"paper_slug":"sequence-level-knowledge-distillation","repo_url":"https://github.com/facebookresearch/stopes","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"sequence-level-knowledge-distillation","repo_url":"https://github.com/ictnlp/Seq-NAT","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"sequence-level-knowledge-distillation","repo_url":"https://github.com/xuanlinli17/autoregressive_inference","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"knowledge-distillation","task_name":"Knowledge Distillation"},{"task_slug":"machine-translation","task_name":"Machine Translation"},{"task_slug":"nmt","task_name":"NMT"},{"task_slug":"translation","task_name":"Translation"}],"methods":[{"method_slug":"knowledge-distillation","method_name":"Knowledge Distillation"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/machine-translation-on-iwslt2015-thai-english","task":"Machine Translation","dataset":"IWSLT2015 Thai-English","model":"Seq-KD + Seq-Inter + Word-KD","rank_in_archive_order":1,"of":1,"metrics":{"BLEU score":"14.2"},"uses_additional_data":false},{"leaderboard":"/sota/machine-translation-on-wmt2014-english-german","task":"Machine Translation","dataset":"WMT2014 English-German","model":"Seq-KD + Seq-Inter + Word-KD","rank_in_archive_order":84,"of":91,"metrics":{"BLEU score":"18.5"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1606.07947","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1606.07947"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/harvardnlp/nmt-android","reach":{"status":"ok","spdx":"GPL-3.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/harvardnlp/seq2seq-attn","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/xuanlinli17/autoregressive_inference","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/anonymouscode115/autoregressive_inference","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ictnlp/Seq-NAT","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/stopes","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":2},"by_repo_kind":{"official":{"samples":2,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"23a7ea7da63c4e61","entry":"get_data","repo":"harvardnlp/seq2seq-attn","repo_kind":"official","path":"preprocess-shards.py","file_url":"https://github.com/harvardnlp/seq2seq-attn/blob/HEAD/preprocess-shards.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"23a7ea7da63c4e61"}},{"code_sha256_prefix":"5eda8017f4a2862c","entry":"pad","repo":"harvardnlp/seq2seq-attn","repo_kind":"official","path":"preprocess-shards.py","file_url":"https://github.com/harvardnlp/seq2seq-attn/blob/HEAD/preprocess-shards.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5eda8017f4a2862c"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}