{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/distilling-linguistic-context-for-language","title":"Distilling Linguistic Context for Language Model Compression","arxiv_id":"2109.08359","date":"2021-09-17","proceeding":"EMNLP 2021 11","authors":["Geondo Park","Gyeongman Kim","Eunho Yang"],"abstract":"A computationally expensive and memory intensive neural network lies behind the recent success of language representation learning. Knowledge distillation, a major technique for deploying such a vast language model in resource-scarce environments, transfers the knowledge on individual word representations learned without restrictions. In this paper, inspired by the recent observations that language representations are relatively positioned and have more semantic knowledge as a whole, we present a new knowledge distillation objective for language representation learning that transfers the contextual knowledge via two types of relationships across representations: Word Relation and Layer Transforming Relation. Unlike other recent distillation techniques for the language models, our contextual distillation does not have any restrictions on architectural changes between teacher and student. We validate the effectiveness of our method on challenging benchmarks of language understanding tasks, not only in architectures of various sizes, but also in combination with DynaBERT, the recently proposed adaptive size pruning method.","url_abs":"https://arxiv.org/abs/2109.08359v1","url_pdf":"https://arxiv.org/pdf/2109.08359v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"distilling-linguistic-context-for-language","repo_url":"https://github.com/geondopark/ckd","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"knowledge-distillation","task_name":"Knowledge Distillation"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"model-compression","task_name":"Model Compression"},{"task_slug":null,"task_name":"Relation"},{"task_slug":"representation-learning","task_name":"Representation Learning"},{"task_slug":"model","task_name":"model"}],"methods":[{"method_slug":"dynabert","method_name":"DynaBERT"},{"method_slug":"knowledge-distillation","method_name":"Knowledge Distillation"},{"method_slug":"pruning","method_name":"Pruning"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2109.08359","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.08359"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/geondopark/ckd","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/GeondoPark/CKD","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":1,"ran_draft_wrong":1,"unverified":9},"by_repo_kind":{"official":{"samples":11,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"b800b685392b45f9","entry":"WR_Dist","repo":"GeondoPark/CKD","repo_kind":"official","path":"loss.py","file_url":"https://github.com/GeondoPark/CKD/blob/HEAD/loss.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b800b685392b45f9"}},{"code_sha256_prefix":"86c8ee1f21537ab4","entry":"batch_pairwise_squared_distance","repo":"GeondoPark/CKD","repo_kind":"official","path":"loss.py","file_url":"https://github.com/GeondoPark/CKD/blob/HEAD/loss.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"86c8ee1f21537ab4"}},{"code_sha256_prefix":"5f120c1844e683ad","entry":"central_squared_distance","repo":"geondopark/ckd","repo_kind":"official","path":"loss.py","file_url":"https://github.com/geondopark/ckd/blob/HEAD/loss.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5f120c1844e683ad"}},{"code_sha256_prefix":"e0dc53f0929c6449","entry":"change_args","repo":"geondopark/ckd","repo_kind":"official","path":"distil_config.py","file_url":"https://github.com/geondopark/ckd/blob/HEAD/distil_config.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e0dc53f0929c6449"}},{"code_sha256_prefix":"fee2c1995f150225","entry":"get_eval_metric","repo":"geondopark/ckd","repo_kind":"official","path":"glue_train.py","file_url":"https://github.com/geondopark/ckd/blob/HEAD/glue_train.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fee2c1995f150225"}},{"code_sha256_prefix":"97abae8d9724c599","entry":"load_squad_dataset","repo":"geondopark/ckd","repo_kind":"official","path":"dataset.py","file_url":"https://github.com/geondopark/ckd/blob/HEAD/dataset.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"97abae8d9724c599"}},{"code_sha256_prefix":"de26a43a3cba0208","entry":"matching_alignment","repo":"geondopark/ckd","repo_kind":"official","path":"utils.py","file_url":"https://github.com/geondopark/ckd/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"de26a43a3cba0208"}},{"code_sha256_prefix":"d0e10e804efa302e","entry":"prepare_embedding_retrieval","repo":"geondopark/ckd","repo_kind":"official","path":"data_augment.py","file_url":"https://github.com/geondopark/ckd/blob/HEAD/data_augment.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d0e10e804efa302e"}},{"code_sha256_prefix":"1894e97f23292978","entry":"set_experiments","repo":"geondopark/ckd","repo_kind":"official","path":"utils.py","file_url":"https://github.com/geondopark/ckd/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1894e97f23292978"}},{"code_sha256_prefix":"22fcb8611ccd6b38","entry":"strip_accents","repo":"geondopark/ckd","repo_kind":"official","path":"data_augment.py","file_url":"https://github.com/geondopark/ckd/blob/HEAD/data_augment.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"22fcb8611ccd6b38"}},{"code_sha256_prefix":"115c66fe58517dd8","entry":"window_index","repo":"geondopark/ckd","repo_kind":"official","path":"loss.py","file_url":"https://github.com/geondopark/ckd/blob/HEAD/loss.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"115c66fe58517dd8"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}