{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/distilling-the-knowledge-in-a-neural-network","title":"Distilling the Knowledge in a Neural Network","arxiv_id":"1503.02531","date":"2015-03-09","proceeding":null,"authors":["Geoffrey Hinton","Oriol Vinyals","Jeff Dean"],"abstract":"A very simple way to improve the performance of almost any machine learning\nalgorithm is to train many different models on the same data and then to\naverage their predictions. Unfortunately, making predictions using a whole\nensemble of models is cumbersome and may be too computationally expensive to\nallow deployment to a large number of users, especially if the individual\nmodels are large neural nets. Caruana and his collaborators have shown that it\nis possible to compress the knowledge in an ensemble into a single model which\nis much easier to deploy and we develop this approach further using a different\ncompression technique. We achieve some surprising results on MNIST and we show\nthat we can significantly improve the acoustic model of a heavily used\ncommercial system by distilling the knowledge in an ensemble of models into a\nsingle model. We also introduce a new type of ensemble composed of one or more\nfull models and many specialist models which learn to distinguish fine-grained\nclasses that the full models confuse. Unlike a mixture of experts, these\nspecialist models can be trained rapidly and in parallel.","url_abs":"http://arxiv.org/abs/1503.02531v1","url_pdf":"http://arxiv.org/pdf/1503.02531v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/Adlik/model_optimizer","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/DunZhang/KnowledgeDistillation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/Irtza/Keras_model_compression","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/JunzWu/Distilling-the-Knowledge-in-a-Neural-Network","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/KaiyuYue/mgd","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/KellyYutongHe/Knowledge-Distillation-Net-with-Swish","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/MasLiang/Learning-without-Forgetting-using-Pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/MinJunKang/CIFAR-100--Knowledge-Distillation-with-Augmented-Data","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/MinJunKang/ImageNet-Knowledge-Distillation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/MinJunKang/Knowledge-Distillation_MNIST","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/NeelayS/Knowledge-Distillation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/TakieddineSOUALHI/Transfer_learning","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/altndrr/persona","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/ankur-98/Multi-domain-Knowledge-Distillation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/aquvitae/aquvitae","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/aryanasadianuoit/distillation_deep","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/chumingqian/Model_Compression_For_YOLOV3-V4","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/elasticdeeplearning/edl","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"paddle","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/franknb/Text-Summarization","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/generation21/generation6011","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/haro-cyber/KnowledgeDistillation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/haysacks/spot-the-coin","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/hunto/image_classification_sota","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/jaychoi12/LG_KD","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/jchen98/compression","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/jpmcd/TensorFlow-KnowledgeDistillation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/kaiyangzhou/on-device-dg","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/kaung-htet-myat/Multi-teachers-Knowledge-Distillation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":null},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/knotgrass/Knowledge-Distillation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/lingyucai/Distilling-the-Knowledge-in-a-Neural-Network","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/markliou/model_distillation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/mckunkel/DistillingObjectDetector","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/mkavim/finetune_bert","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":null},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/paddlepaddle/edl","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"paddle","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/pra-dan/Worth-Reading-ML-Papers","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/robertjkeck2/EmoNet","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/robertjkeck2/EmoTe","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/robertjkeck2/pwdl","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/rohin-dasari/Tensorflow-Knowledge-Distillation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/satyakees/FaultNet","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/scy-x/d3still","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/see--/speech_recognition","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/shinome551/select_with_distillation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/shriramsb/Distilling-the-Knowledge-in-a-Neural-Network","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/stdereka/knowledge-distillation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/sungnyun/LG-knowledge-distillation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/t0nyliang/EEGMobile","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/thaonguyen19/modeldistillation-pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/trqminh/knowledge-distillation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/varungumma/fairseq","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/vrvlive/knowlege-distillation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/wannabeOG/ExpertNet-Pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/wonbeomjang/Knowledge-Distilling-PyTorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/yangze01/Distilling_the_Knowledge_in_a_Neural_Network","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/yangze01/Distilling_the_Knowledge_in_a_Neural_Network_pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/yell/kaggle-camera","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/yoshitomo-matsubara/torchdistill","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/yurangja99/Pytorch-Sketch-RNN","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"unanswered"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/yzd-v/cls_KD","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/zh320/realtime-semantic-segmentation-pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/Deci-AI/super-gradients","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":null},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/alibaba/EasyNLP/tree/master/examples/knowledge_distillation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"jax","reach":null},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/labmlai/annotated_deep_learning_paper_implementations","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":null},{"paper_slug":"distilling-the-knowledge-in-a-neural-network","repo_url":"https://github.com/yangyucheng000/Papers/tree/main/DistillExplanation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null}],"tasks":[{"task_slug":"knowledge-distillation","task_name":"Knowledge Distillation"},{"task_slug":"mixture-of-experts","task_name":"Mixture-of-Experts"}],"methods":[{"method_slug":"knowledge-distillation","method_name":"Knowledge Distillation"}],"datasets_introduced":[],"methods_introduced":[{"slug":"knowledge-distillation","name":"Knowledge Distillation","full_name":"Knowledge Distillation"}],"results":[{"leaderboard":"/sota/knowledge-distillation-on-cifar-100","task":"Knowledge Distillation","dataset":"CIFAR-100","model":"resnet8x4 (T: resnet32x4 S: resnet8x4)","rank_in_archive_order":20,"of":27,"metrics":{"Top-1 Accuracy (%)":"73.33"},"uses_additional_data":false},{"leaderboard":"/sota/knowledge-distillation-on-cifar-100","task":"Knowledge Distillation","dataset":"CIFAR-100","model":"vgg8 (T:vgg13 S:vgg8)","rank_in_archive_order":21,"of":27,"metrics":{"Top-1 Accuracy (%)":"72.98"},"uses_additional_data":false},{"leaderboard":"/sota/knowledge-distillation-on-cifar-100","task":"Knowledge Distillation","dataset":"CIFAR-100","model":"resnet110 (T:resnet110 S:resnet20)","rank_in_archive_order":27,"of":27,"metrics":{"Top-1 Accuracy (%)":"70.67"},"uses_additional_data":false},{"leaderboard":"/sota/knowledge-distillation-on-imagenet","task":"Knowledge Distillation","dataset":"ImageNet","model":"ADLIK-MO(T: ResNet101 S: ResNet50)","rank_in_archive_order":22,"of":52,"metrics":{"CRD training setting":"✘","Top-1 accuracy %":"77.14","model size":"99M"},"uses_additional_data":false},{"leaderboard":"/sota/knowledge-distillation-on-imagenet","task":"Knowledge Distillation","dataset":"ImageNet","model":"KD (T: ResNet-34 S:ResNet-18)","rank_in_archive_order":50,"of":52,"metrics":{"CRD training setting":"✓","Top-1 accuracy %":"70.66"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/1503.02531","atlas_url":"https://app.syntology.ai/?focus=1503.02531","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1503.02531"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yoshitomo-matsubara/torchdistill","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/haysacks/spot-the-coin","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/haro-cyber/KnowledgeDistillation","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ankur-98/Multi-domain-Knowledge-Distillation","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/robertjkeck2/pwdl","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yzd-v/cls_KD","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/paddlepaddle/edl","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/wannabeOG/ExpertNet-Pytorch","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/hunto/image_classification_sota","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/KaiyuYue/mgd","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kaung-htet-myat/Multi-teachers-Knowledge-Distillation","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yell/kaggle-camera","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jaychoi12/LG_KD","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/MinJunKang/ImageNet-Knowledge-Distillation","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/altndrr/persona","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/trqminh/knowledge-distillation","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yangyucheng000/Papers/tree/main/DistillExplanation","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/satyakees/FaultNet","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/scy-x/d3still","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/pra-dan/Worth-Reading-ML-Papers","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/lingyucai/Distilling-the-Knowledge-in-a-Neural-Network","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Adlik/model_optimizer","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/NeelayS/Knowledge-Distillation","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/chumingqian/Model_Compression_For_YOLOV3-V4","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/KellyYutongHe/Knowledge-Distillation-Net-with-Swish","reach":null}],"summary":{"ran_fixture":1,"ran_honours":6,"ran_draft_wrong":8,"ran":1,"unverified":21},"by_repo_kind":{"listed":{"samples":36,"ran":16,"repositories":14}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":10,"samples":[{"code_sha256_prefix":"a427a6169de8f607","entry":"compute_accuracy","repo":"yoshitomo-matsubara/torchdistill","repo_kind":"listed","path":"examples/torchvision/image_classification.py","file_url":"https://github.com/yoshitomo-matsubara/torchdistill/blob/HEAD/examples/torchvision/image_classification.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a427a6169de8f607"}},{"code_sha256_prefix":"2cbf06e40dbf54ee","entry":"compute_mean_and_std","repo":"MasLiang/Learning-without-Forgetting-using-Pytorch","repo_kind":"listed","path":"dataset.py","file_url":"https://github.com/MasLiang/Learning-without-Forgetting-using-Pytorch/blob/HEAD/dataset.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2cbf06e40dbf54ee"}},{"code_sha256_prefix":"0a330946ecfac712","entry":"conv_block","repo":"wonbeomjang/Knowledge-Distilling-PyTorch","repo_kind":"listed","path":"models.py","file_url":"https://github.com/wonbeomjang/Knowledge-Distilling-PyTorch/blob/HEAD/models.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0a330946ecfac712"}},{"code_sha256_prefix":"274528b19a06a729","entry":"d3_loss","repo":"scy-x/d3still","repo_kind":"listed","path":"AIR_Distiller/distillers/D3.py","file_url":"https://github.com/scy-x/d3still/blob/HEAD/AIR_Distiller/distillers/D3.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"274528b19a06a729"}},{"code_sha256_prefix":"7308ca9b35a3b212","entry":"eval_epoch","repo":"stdereka/knowledge-distillation","repo_kind":"listed","path":"training.py","file_url":"https://github.com/stdereka/knowledge-distillation/blob/HEAD/training.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7308ca9b35a3b212"}},{"code_sha256_prefix":"7ee88cf415894459","entry":"fetch_teacher_outputs","repo":"KellyYutongHe/Knowledge-Distillation-Net-with-Swish","repo_kind":"listed","path":"kd.py","file_url":"https://github.com/KellyYutongHe/Knowledge-Distillation-Net-with-Swish/blob/HEAD/kd.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7ee88cf415894459"}},{"code_sha256_prefix":"62f639fec5cb08e5","entry":"fit_epoch","repo":"stdereka/knowledge-distillation","repo_kind":"listed","path":"training.py","file_url":"https://github.com/stdereka/knowledge-distillation/blob/HEAD/training.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"62f639fec5cb08e5"}},{"code_sha256_prefix":"8a5635108617215c","entry":"loss_fn_kd","repo":"KellyYutongHe/Knowledge-Distillation-Net-with-Swish","repo_kind":"listed","path":"kd.py","file_url":"https://github.com/KellyYutongHe/Knowledge-Distillation-Net-with-Swish/blob/HEAD/kd.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8a5635108617215c"}},{"code_sha256_prefix":"9c20329610d8c5e5","entry":"obtain_avg_forward_time","repo":"chumingqian/Model_Compression_For_YOLOV3-V4","repo_kind":"listed","path":"normal_prune.py","file_url":"https://github.com/chumingqian/Model_Compression_For_YOLOV3-V4/blob/HEAD/normal_prune.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"none","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9c20329610d8c5e5"}},{"code_sha256_prefix":"acb7b29614f1a161","entry":"obtain_filters_mask","repo":"chumingqian/Model_Compression_For_YOLOV3-V4","repo_kind":"listed","path":"normal_prune.py","file_url":"https://github.com/chumingqian/Model_Compression_For_YOLOV3-V4/blob/HEAD/normal_prune.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"acb7b29614f1a161"}},{"code_sha256_prefix":"0bd196791b1b86e6","entry":"output_adaptor","repo":"DunZhang/KnowledgeDistillation","repo_kind":"listed","path":"Examples/example_multi_layer_based_model/distill_bert.py","file_url":"https://github.com/DunZhang/KnowledgeDistillation/blob/HEAD/Examples/example_multi_layer_based_model/distill_bert.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"0bd196791b1b86e6"}},{"code_sha256_prefix":"bee766c6e15044c2","entry":"preprocess_input","repo":"mckunkel/DistillingObjectDetector","repo_kind":"listed","path":"models/microxception.py","file_url":"https://github.com/mckunkel/DistillingObjectDetector/blob/HEAD/models/microxception.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bee766c6e15044c2"}},{"code_sha256_prefix":"ec78bec91ac01396","entry":"prune_and_eval","repo":"chumingqian/Model_Compression_For_YOLOV3-V4","repo_kind":"listed","path":"normal_prune.py","file_url":"https://github.com/chumingqian/Model_Compression_For_YOLOV3-V4/blob/HEAD/normal_prune.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ec78bec91ac01396"}},{"code_sha256_prefix":"cb5cbea0e8219ff5","entry":"softmax","repo":"knotgrass/Knowledge-Distillation","repo_kind":"listed","path":"distiller/loss.py","file_url":"https://github.com/knotgrass/Knowledge-Distillation/blob/HEAD/distiller/loss.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cb5cbea0e8219ff5"}},{"code_sha256_prefix":"00a58257f6886584","entry":"train","repo":"stdereka/knowledge-distillation","repo_kind":"listed","path":"training.py","file_url":"https://github.com/stdereka/knowledge-distillation/blob/HEAD/training.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"00a58257f6886584"}},{"code_sha256_prefix":"95712f5d0bd66d34","entry":"train_data_adaptor","repo":"DunZhang/KnowledgeDistillation","repo_kind":"listed","path":"Examples/example_multi_layer_based_model/distill_bert.py","file_url":"https://github.com/DunZhang/KnowledgeDistillation/blob/HEAD/Examples/example_multi_layer_based_model/distill_bert.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"95712f5d0bd66d34"}},{"code_sha256_prefix":"f0854de100bbea76","entry":"SqueezeNet","repo":"mckunkel/DistillingObjectDetector","repo_kind":"listed","path":"models/squeezenet.py","file_url":"https://github.com/mckunkel/DistillingObjectDetector/blob/HEAD/models/squeezenet.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f0854de100bbea76"}},{"code_sha256_prefix":"6300eb91406d21bf","entry":"SqueezeNet","repo":"mckunkel/DistillingObjectDetector","repo_kind":"listed","path":"models/squeezenet_model.py","file_url":"https://github.com/mckunkel/DistillingObjectDetector/blob/HEAD/models/squeezenet_model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6300eb91406d21bf"}},{"code_sha256_prefix":"9b8289076669fe4f","entry":"accuracy","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"unverified","verification_level":0,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"9b8289076669fe4f"}},{"code_sha256_prefix":"77551a1423d80be3","entry":"albumen_loader","repo":"knotgrass/Knowledge-Distillation","repo_kind":"listed","path":"distiller/datasets.py","file_url":"https://github.com/knotgrass/Knowledge-Distillation/blob/HEAD/distiller/datasets.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"77551a1423d80be3"}},{"code_sha256_prefix":"00c0d849e940be32","entry":"check_same_order","repo":"knotgrass/Knowledge-Distillation","repo_kind":"listed","path":"distiller/sorted_same_order.py","file_url":"https://github.com/knotgrass/Knowledge-Distillation/blob/HEAD/distiller/sorted_same_order.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"00c0d849e940be32"}},{"code_sha256_prefix":"f14f766b7e289b16","entry":"conv2d_bn","repo":"mckunkel/DistillingObjectDetector","repo_kind":"listed","path":"models/inceptionV3.py","file_url":"https://github.com/mckunkel/DistillingObjectDetector/blob/HEAD/models/inceptionV3.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f14f766b7e289b16"}},{"code_sha256_prefix":"671c7f056497a8b6","entry":"desc","repo":"knotgrass/Knowledge-Distillation","repo_kind":"listed","path":"distiller/print_utils.py","file_url":"https://github.com/knotgrass/Knowledge-Distillation/blob/HEAD/distiller/print_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"671c7f056497a8b6"}},{"code_sha256_prefix":"1b249ba85d8cf4ba","entry":"fire_module","repo":"mckunkel/DistillingObjectDetector","repo_kind":"listed","path":"models/squeezenet.py","file_url":"https://github.com/mckunkel/DistillingObjectDetector/blob/HEAD/models/squeezenet.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1b249ba85d8cf4ba"}},{"code_sha256_prefix":"978081881d93f4be","entry":"get_mobilenet","repo":"mckunkel/DistillingObjectDetector","repo_kind":"listed","path":"models/mobilenet.py","file_url":"https://github.com/mckunkel/DistillingObjectDetector/blob/HEAD/models/mobilenet.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"978081881d93f4be"}},{"code_sha256_prefix":"a8b17275da494ca9","entry":"get_order","repo":"knotgrass/Knowledge-Distillation","repo_kind":"listed","path":"distiller/sorted_same_order.py","file_url":"https://github.com/knotgrass/Knowledge-Distillation/blob/HEAD/distiller/sorted_same_order.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a8b17275da494ca9"}},{"code_sha256_prefix":"240a91c16bc48dd5","entry":"loss_fn_kd","repo":"knotgrass/Knowledge-Distillation","repo_kind":"listed","path":"distiller/loss.py","file_url":"https://github.com/knotgrass/Knowledge-Distillation/blob/HEAD/distiller/loss.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"240a91c16bc48dd5"}},{"code_sha256_prefix":"9f00b79fbcd75706","entry":"mean_std","repo":"knotgrass/Knowledge-Distillation","repo_kind":"listed","path":"distiller/pseudo_label.py","file_url":"https://github.com/knotgrass/Knowledge-Distillation/blob/HEAD/distiller/pseudo_label.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9f00b79fbcd75706"}},{"code_sha256_prefix":"2d50b69c5762cfbe","entry":"microXception","repo":"mckunkel/DistillingObjectDetector","repo_kind":"listed","path":"models/microxception.py","file_url":"https://github.com/mckunkel/DistillingObjectDetector/blob/HEAD/models/microxception.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2d50b69c5762cfbe"}},{"code_sha256_prefix":"3ec85b13b3da58a4","entry":"miniXception","repo":"mckunkel/DistillingObjectDetector","repo_kind":"listed","path":"models/minixception.py","file_url":"https://github.com/mckunkel/DistillingObjectDetector/blob/HEAD/models/minixception.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3ec85b13b3da58a4"}},{"code_sha256_prefix":"9eb8ac7c03e9b11b","entry":"preprocess_input","repo":"mckunkel/DistillingObjectDetector","repo_kind":"listed","path":"models/squeezenet.py","file_url":"https://github.com/mckunkel/DistillingObjectDetector/blob/HEAD/models/squeezenet.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9eb8ac7c03e9b11b"}},{"code_sha256_prefix":"37db7f404db2f6ab","entry":"sort_by_order","repo":"knotgrass/Knowledge-Distillation","repo_kind":"listed","path":"distiller/sorted_same_order.py","file_url":"https://github.com/knotgrass/Knowledge-Distillation/blob/HEAD/distiller/sorted_same_order.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"37db7f404db2f6ab"}},{"code_sha256_prefix":"f7005e3d307e1c00","entry":"train","repo":"knotgrass/Knowledge-Distillation","repo_kind":"listed","path":"distiller/distiller.py","file_url":"https://github.com/knotgrass/Knowledge-Distillation/blob/HEAD/distiller/distiller.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f7005e3d307e1c00"}},{"code_sha256_prefix":"c3041253233293ca","entry":"train_body","repo":"yangze01/Distilling_the_Knowledge_in_a_Neural_Network_pytorch","repo_kind":"listed","path":"joint_main.py","file_url":"https://github.com/yangze01/Distilling_the_Knowledge_in_a_Neural_Network_pytorch/blob/HEAD/joint_main.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c3041253233293ca"}},{"code_sha256_prefix":"b7e8ebfd9b3a5233","entry":"train_student_model","repo":"trqminh/knowledge-distillation","repo_kind":"listed","path":"train/train_student.py","file_url":"https://github.com/trqminh/knowledge-distillation/blob/HEAD/train/train_student.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b7e8ebfd9b3a5233"}},{"code_sha256_prefix":"41bcf9c6e97f30d5","entry":"validate","repo":"KaiyuYue/mgd","repo_kind":"listed","path":"cls/main_mgd.py","file_url":"https://github.com/KaiyuYue/mgd/blob/HEAD/cls/main_mgd.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"41bcf9c6e97f30d5"}},{"code_sha256_prefix":"5b365f68a2be17aa","entry":"validate","repo":"JunzWu/Distilling-the-Knowledge-in-a-Neural-Network","repo_kind":"listed","path":"main_imagenet.py","file_url":"https://github.com/JunzWu/Distilling-the-Knowledge-in-a-Neural-Network/blob/HEAD/main_imagenet.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5b365f68a2be17aa"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}