{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/how-to-distill-your-bert-an-empirical-study","title":"How to Distill your BERT: An Empirical Study on the Impact of Weight Initialisation and Distillation Objectives","arxiv_id":"2305.15032","date":"2023-05-24","proceeding":null,"authors":["Xinpeng Wang","Leonie Weissweiler","Hinrich Schütze","Barbara Plank"],"abstract":"Recently, various intermediate layer distillation (ILD) objectives have been shown to improve compression of BERT models via Knowledge Distillation (KD). However, a comprehensive evaluation of the objectives in both task-specific and task-agnostic settings is lacking. To the best of our knowledge, this is the first work comprehensively evaluating distillation objectives in both settings. We show that attention transfer gives the best performance overall. We also study the impact of layer choice when initializing the student from the teacher layers, finding a significant impact on the performance in task-specific distillation. For vanilla KD and hidden states transfer, initialisation with lower layers of the teacher gives a considerable improvement over higher layers, especially on the task of QNLI (up to an absolute percentage change of 17.8 in accuracy). Attention transfer behaves consistently under different initialisation settings. We release our code as an efficient transformer-based model distillation framework for further studies.","url_abs":"https://arxiv.org/abs/2305.15032v1","url_pdf":"https://arxiv.org/pdf/2305.15032v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"how-to-distill-your-bert-an-empirical-study","repo_url":"https://github.com/mainlp/how-to-distill-your-bert","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"knowledge-distillation","task_name":"Knowledge Distillation"},{"task_slug":null,"task_name":"QNLI"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"knowledge-distillation","method_name":"Knowledge Distillation"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2305.15032","atlas_url":"https://app.syntology.ai/?focus=2305.15032","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.15032"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mainlp/how-to-distill-your-bert","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/mainlp/How-to-distill-your-BERT","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":1,"ran_draft_wrong":1,"unverified":9},"by_repo_kind":{"official":{"samples":11,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"76a475fa13f37cec","entry":"get_valid_dataloader","repo":"mainlp/How-to-distill-your-BERT","repo_kind":"official","path":"task_agnostic_distillation/run_pretraining.py","file_url":"https://github.com/mainlp/How-to-distill-your-BERT/blob/HEAD/task_agnostic_distillation/run_pretraining.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"76a475fa13f37cec"}},{"code_sha256_prefix":"de41371cecbab67c","entry":"task_specific_config","repo":"mainlp/how-to-distill-your-bert","repo_kind":"official","path":"task_specific_distillation/experiments.py","file_url":"https://github.com/mainlp/how-to-distill-your-bert/blob/HEAD/task_specific_distillation/experiments.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"de41371cecbab67c"}},{"code_sha256_prefix":"36f7375b15089e34","entry":"att_kl","repo":"mainlp/How-to-distill-your-BERT","repo_kind":"official","path":"task_agnostic_distillation/methods/feature_distill.py","file_url":"https://github.com/mainlp/How-to-distill-your-BERT/blob/HEAD/task_agnostic_distillation/methods/feature_distill.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"36f7375b15089e34"}},{"code_sha256_prefix":"0888d8eb55277184","entry":"att_val_kl","repo":"mainlp/How-to-distill-your-BERT","repo_kind":"official","path":"task_agnostic_distillation/methods/feature_distill.py","file_url":"https://github.com/mainlp/How-to-distill-your-BERT/blob/HEAD/task_agnostic_distillation/methods/feature_distill.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0888d8eb55277184"}},{"code_sha256_prefix":"f114723b758d8789","entry":"bias_gelu","repo":"mainlp/How-to-distill-your-BERT","repo_kind":"official","path":"task_agnostic_distillation/pretraining/modeling.py","file_url":"https://github.com/mainlp/How-to-distill-your-BERT/blob/HEAD/task_agnostic_distillation/pretraining/modeling.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f114723b758d8789"}},{"code_sha256_prefix":"c5070fa84fd2b490","entry":"cosine_similarity","repo":"mainlp/How-to-distill-your-BERT","repo_kind":"official","path":"task_agnostic_distillation/methods/pear_loss.py","file_url":"https://github.com/mainlp/How-to-distill-your-BERT/blob/HEAD/task_agnostic_distillation/methods/pear_loss.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c5070fa84fd2b490"}},{"code_sha256_prefix":"e4c865c0ef8d62b4","entry":"data_aug","repo":"mainlp/How-to-distill-your-BERT","repo_kind":"official","path":"task_agnostic_distillation/methods/feature_distill.py","file_url":"https://github.com/mainlp/How-to-distill-your-BERT/blob/HEAD/task_agnostic_distillation/methods/feature_distill.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e4c865c0ef8d62b4"}},{"code_sha256_prefix":"2fd4f78b58e81a10","entry":"f_gelu","repo":"mainlp/How-to-distill-your-BERT","repo_kind":"official","path":"task_agnostic_distillation/pretraining/modeling.py","file_url":"https://github.com/mainlp/How-to-distill-your-BERT/blob/HEAD/task_agnostic_distillation/pretraining/modeling.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2fd4f78b58e81a10"}},{"code_sha256_prefix":"87b054b1860fbd9e","entry":"get_deepspeed_config","repo":"mainlp/How-to-distill-your-BERT","repo_kind":"official","path":"task_agnostic_distillation/pretraining/modeling.py","file_url":"https://github.com/mainlp/How-to-distill-your-BERT/blob/HEAD/task_agnostic_distillation/pretraining/modeling.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"87b054b1860fbd9e"}},{"code_sha256_prefix":"cda3d1271f46dae7","entry":"inter_class_relation","repo":"mainlp/How-to-distill-your-BERT","repo_kind":"official","path":"task_agnostic_distillation/methods/pear_loss.py","file_url":"https://github.com/mainlp/How-to-distill-your-BERT/blob/HEAD/task_agnostic_distillation/methods/pear_loss.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cda3d1271f46dae7"}},{"code_sha256_prefix":"78c3c61943df0cc0","entry":"pearson_correlation","repo":"mainlp/How-to-distill-your-BERT","repo_kind":"official","path":"task_agnostic_distillation/methods/pear_loss.py","file_url":"https://github.com/mainlp/How-to-distill-your-BERT/blob/HEAD/task_agnostic_distillation/methods/pear_loss.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"78c3c61943df0cc0"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}