{"url":"/task/knowledge-distillation","name":"Knowledge Distillation","slug":"knowledge-distillation","description_markdown":"Knowledge distillation is the process of transferring knowledge from a large model to a smaller one. While large models (such as very deep neural networks or ensembles of many models) have higher knowledge capacity than small models, this capacity might not be fully utilized.","categories":[{"name":"Computer Vision","url":"/area/computer-vision"},{"name":"Natural Language Processing","url":"/area/natural-language-processing"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":4240,"papers_with_code":1740,"benchmarks":7,"benchmark_tables_in_archive":11,"benchmark_tables_shown":11,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":6,"subtasks":2,"parent_tasks":0},"benchmarks":[{"leaderboard":"/sota/knowledge-distillation-on-imagenet","slug":"knowledge-distillation-on-imagenet","dataset":"ImageNet","dataset_url":"/dataset/imagenet","rows_in_archive":52,"metrics":["Top-1 accuracy %","model size","CRD training setting"],"first_row_in_archive_order":{"model":"ScaleKD (T:BEiT-L S:ViT-B/14)","paper_title":"ScaleKD: Strong Vision Transformers Could Be Excellent Teachers","paper_url":"/paper/scalekd-strong-vision-transformers-could-be","paper_date":"2024-11-11","arxiv_id":"2411.06786","code_links":[{"title":"deep-optimization/scalekd","url":"https://github.com/deep-optimization/scalekd"}],"syntology":null}},{"leaderboard":"/sota/knowledge-distillation-on-cifar-100","slug":"knowledge-distillation-on-cifar-100","dataset":"CIFAR-100","dataset_url":"/dataset/cifar-100","rows_in_archive":27,"metrics":["Top-1 Accuracy (%)"],"first_row_in_archive_order":{"model":"SRD (T:resnet-32x4, S:shufflenet-v2)","paper_title":"Understanding the Role of the Projector in Knowledge Distillation","paper_url":"/paper/a-closer-look-at-the-training-dynamics-of","paper_date":"2023-03-20","arxiv_id":"2303.11098","code_links":[{"title":"yoshitomo-matsubara/torchdistill","url":"https://github.com/yoshitomo-matsubara/torchdistill"},{"title":"Hazqeel09/ellzaf_ml","url":"https://github.com/Hazqeel09/ellzaf_ml"},{"title":"roymiles/vkd","url":"https://github.com/roymiles/vkd"},{"title":"roymiles/simple-recipe-distillation","url":"https://github.com/roymiles/simple-recipe-distillation"}],"syntology":null}},{"leaderboard":"/sota/knowledge-distillation-on-coco","slug":"knowledge-distillation-on-coco","dataset":"COCO (Common Objects in Context)","dataset_url":"/dataset/coco","rows_in_archive":4,"metrics":[" box AP","mask AP","mAP"],"first_row_in_archive_order":{"model":"ADLIK-Faster (T: Faster R-CNN vit-base S: Faster R-CNN deit-small)","paper_title":"Focal and Global Knowledge Distillation for Detectors","paper_url":"/paper/focal-and-global-knowledge-distillation-for","paper_date":"2021-11-23","arxiv_id":"2111.11837","code_links":[{"title":"yzd-v/FGD","url":"https://github.com/yzd-v/FGD"}],"syntology":null}},{"leaderboard":"/sota/knowledge-distillation-on-coco-2017-val","slug":"knowledge-distillation-on-coco-2017-val","dataset":"COCO 2017 val","dataset_url":null,"rows_in_archive":3,"metrics":["AP@0.5","AP@0.75","mAP"],"first_row_in_archive_order":{"model":"ReviewKD++(T: faster rcnn(resnet101), S:faster rcnn(resnet50))","paper_title":"Improving Knowledge Distillation via Regularizing Feature Norm and Direction","paper_url":"/paper/improving-knowledge-distillation-via-1","paper_date":"2023-05-26","arxiv_id":"2305.17007","code_links":[{"title":"wangyz1608/knowledge-distillation-via-nd","url":"https://github.com/wangyz1608/knowledge-distillation-via-nd"}],"syntology":{"n":3,"n_ran":3,"n_unverified":0,"n_pointer_only":3}}},{"leaderboard":"/sota/knowledge-distillation-on-pascal-voc","slug":"knowledge-distillation-on-pascal-voc","dataset":"PASCAL VOC","dataset_url":"/dataset/pascal-voc","rows_in_archive":2,"metrics":["mAP"],"first_row_in_archive_order":{"model":"LSHFM (T: ResNet101 S: ResNet50)","paper_title":"Distilling Knowledge by Mimicking Features","paper_url":"/paper/in-defense-of-feature-mimicking-for-knowledge","paper_date":"2020-11-03","arxiv_id":"2011.01424","code_links":[{"title":"DoctorKey/LSHFM.singleclassification","url":"https://github.com/DoctorKey/LSHFM.singleclassification"},{"title":"DoctorKey/LSHFM.detection","url":"https://github.com/DoctorKey/LSHFM.detection"},{"title":"DoctorKey/LSHFM.multiclassification","url":"https://github.com/DoctorKey/LSHFM.multiclassification"}],"syntology":{"n":6,"n_ran":5,"n_unverified":1,"n_pointer_only":6}}},{"leaderboard":"/sota/knowledge-distillation-on-cityscapes","slug":"knowledge-distillation-on-cityscapes","dataset":"Cityscapes","dataset_url":"/dataset/cityscapes","rows_in_archive":1,"metrics":["AP"],"first_row_in_archive_order":{"model":"CAST","paper_title":"CAST: Contrastive Adaptation and Distillation for Semi-Supervised Instance Segmentation","paper_url":"/paper/cast-contrastive-adaptation-and-distillation","paper_date":"2025-05-28","arxiv_id":"2505.21904","code_links":[],"syntology":null}},{"leaderboard":"/sota/knowledge-distillation-on-kitti","slug":"knowledge-distillation-on-kitti","dataset":"KITTI","dataset_url":"/dataset/kitti","rows_in_archive":1,"metrics":["RMSE","model size"],"first_row_in_archive_order":{"model":"TIE-KD (T: Adabins S: MobileNetV2)","paper_title":"TIE-KD: Teacher-Independent and Explainable Knowledge Distillation for Monocular Depth Estimation","paper_url":"/paper/tie-kd-teacher-independent-and-explainable","paper_date":"2024-02-22","arxiv_id":"2402.14340","code_links":[{"title":"hpc-lab-koreatech/tie-kd","url":"https://github.com/hpc-lab-koreatech/tie-kd"}],"syntology":null}},{"leaderboard":null,"slug":"knowledge-distillation-on-big-content","dataset":"big content","dataset_url":null,"rows_in_archive":0,"metrics":["Negative Mse"],"first_row_in_archive_order":null},{"leaderboard":null,"slug":"knowledge-distillation-on-en-es","dataset":"en es","dataset_url":null,"rows_in_archive":0,"metrics":["Negative Mse"],"first_row_in_archive_order":null},{"leaderboard":null,"slug":"knowledge-distillation-on-en-pt-br","dataset":"en pt br","dataset_url":null,"rows_in_archive":0,"metrics":["Negative Mse"],"first_row_in_archive_order":null},{"leaderboard":null,"slug":"knowledge-distillation-on-small-content","dataset":"small content","dataset_url":null,"rows_in_archive":0,"metrics":["Negative Mse"],"first_row_in_archive_order":null}],"datasets":[{"url":"/dataset/imagenet","name":"ImageNet","full_name":"","num_papers_in_archive":15430},{"url":"/dataset/coco","name":"COCO (Common Objects in Context)","full_name":"Common Objects in Context","num_papers_in_archive":11922},{"url":"/dataset/cifar-100","name":"CIFAR-100","full_name":"","num_papers_in_archive":9045},{"url":"/dataset/cityscapes","name":"Cityscapes","full_name":"","num_papers_in_archive":3702},{"url":"/dataset/kitti","name":"KITTI","full_name":"","num_papers_in_archive":3661},{"url":"/dataset/pascal-voc","name":"PASCAL VOC","full_name":"PASCAL Visual Object Classes Challenge","num_papers_in_archive":198}],"subtasks":[{"url":"/task/data-free-knowledge-distillation","name":"Data-free Knowledge Distillation"},{"url":"/task/self-knowledge-distillation","name":"Self-Knowledge Distillation"}],"parent_tasks":[],"papers":{"order":"repositories listed in the archive (desc), then date (desc); the archive holds no stars","population":"papers tagged with this task that list at least one repository in the archive","shown":30,"of":1740,"tagged_in_all":4240,"items":[{"url":"/paper/focal-loss-for-dense-object-detection","title":"Focal Loss for Dense Object Detection","date":"2017-08-07","arxiv_id":"1708.02002","repositories_listed":234,"syntology":{"n":11,"n_ran":11,"n_unverified":0,"n_pointer_only":6}},{"url":"/paper/distilling-the-knowledge-in-a-neural-network","title":"Distilling the Knowledge in a Neural Network","date":"2015-03-09","arxiv_id":"1503.02531","repositories_listed":64,"syntology":{"n":37,"n_ran":15,"n_unverified":22,"n_pointer_only":10}},{"url":"/paper/well-read-students-learn-better-the-impact-of","title":"Well-Read Students Learn Better: On the Importance of Pre-training Compact Models","date":"2019-08-23","arxiv_id":"1908.08962","repositories_listed":40,"syntology":{"n":32,"n_ran":6,"n_unverified":26,"n_pointer_only":0}},{"url":"/paper/fastspeech-2-fast-and-high-quality-end-to-end","title":"FastSpeech 2: Fast and High-Quality End-to-End Text to Speech","date":"2020-06-08","arxiv_id":"2006.04558","repositories_listed":37,"syntology":{"n":119,"n_ran":73,"n_unverified":46,"n_pointer_only":33}},{"url":"/paper/distilbert-a-distilled-version-of-bert","title":"DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter","date":"2019-10-02","arxiv_id":"1910.01108","repositories_listed":37,"syntology":{"n":27,"n_ran":19,"n_unverified":8,"n_pointer_only":0}},{"url":"/paper/grad-cam-improved-visual-explanations-for","title":"Grad-CAM++: Improved Visual Explanations for Deep Convolutional Networks","date":"2017-10-30","arxiv_id":"1710.11063","repositories_listed":24,"syntology":{"n":7,"n_ran":4,"n_unverified":3,"n_pointer_only":3}},{"url":"/paper/making-monolingual-sentence-embeddings","title":"Making Monolingual Sentence Embeddings Multilingual using Knowledge Distillation","date":"2020-04-21","arxiv_id":"2004.09813","repositories_listed":12,"syntology":{"n":1,"n_ran":0,"n_unverified":1,"n_pointer_only":0}},{"url":"/paper/knowledge-distillation-a-good-teacher-is","title":"Knowledge distillation: A good teacher is patient and consistent","date":"2021-06-09","arxiv_id":"2106.05237","repositories_listed":10,"syntology":null},{"url":"/paper/190910351","title":"TinyBERT: Distilling BERT for Natural Language Understanding","date":"2019-09-23","arxiv_id":"1909.10351","repositories_listed":10,"syntology":{"n":4,"n_ran":0,"n_unverified":4,"n_pointer_only":4}},{"url":"/paper/alphapose-whole-body-regional-multi-person","title":"AlphaPose: Whole-Body Regional Multi-Person Pose Estimation and Tracking in Real-Time","date":"2022-11-07","arxiv_id":"2211.03375","repositories_listed":8,"syntology":null},{"url":"/paper/distilling-knowledge-via-knowledge-review","title":"Distilling Knowledge via Knowledge Review","date":"2021-04-19","arxiv_id":"2104.09044","repositories_listed":7,"syntology":{"n":8,"n_ran":5,"n_unverified":3,"n_pointer_only":5}},{"url":"/paper/fedmd-heterogenous-federated-learning-via","title":"FedMD: Heterogenous Federated Learning via Model Distillation","date":"2019-10-08","arxiv_id":"1910.03581","repositories_listed":7,"syntology":{"n":1,"n_ran":1,"n_unverified":0,"n_pointer_only":1}},{"url":"/paper/paying-more-attention-to-attention-improving","title":"Paying More Attention to Attention: Improving the Performance of Convolutional Neural Networks via Attention Transfer","date":"2016-12-12","arxiv_id":"1612.03928","repositories_listed":7,"syntology":{"n":1,"n_ran":1,"n_unverified":0,"n_pointer_only":1}},{"url":"/paper/sequence-level-knowledge-distillation","title":"Sequence-Level Knowledge Distillation","date":"2016-06-25","arxiv_id":"1606.07947","repositories_listed":6,"syntology":{"n":2,"n_ran":0,"n_unverified":2,"n_pointer_only":0}},{"url":"/paper/awesome-multi-modal-object-tracking","title":"Awesome Multi-modal Object Tracking","date":"2024-05-23","arxiv_id":"2405.14200","repositories_listed":5,"syntology":null},{"url":"/paper/distillm-towards-streamlined-distillation-for","title":"DistiLLM: Towards Streamlined Distillation for Large Language Models","date":"2024-02-06","arxiv_id":"2402.03898","repositories_listed":5,"syntology":{"n":5,"n_ran":5,"n_unverified":0,"n_pointer_only":4}},{"url":"/paper/anomaly-detection-via-reverse-distillation","title":"Anomaly Detection via Reverse Distillation from One-Class Embedding","date":"2022-01-26","arxiv_id":"2201.10703","repositories_listed":5,"syntology":{"n":18,"n_ran":14,"n_unverified":4,"n_pointer_only":15}},{"url":"/paper/ternarybert-distillation-aware-ultra-low-bit","title":"TernaryBERT: Distillation-aware Ultra-low Bit BERT","date":"2020-09-27","arxiv_id":"2009.12812","repositories_listed":5,"syntology":null},{"url":"/paper/proselflc-progressive-self-label-correction","title":"ProSelfLC: Progressive Self Label Correction for Training Robust Deep Neural Networks","date":"2020-05-07","arxiv_id":"2005.03788","repositories_listed":5,"syntology":{"n":11,"n_ran":1,"n_unverified":10,"n_pointer_only":0}},{"url":"/paper/patient-knowledge-distillation-for-bert-model","title":"Patient Knowledge Distillation for BERT Model Compression","date":"2019-08-25","arxiv_id":"1908.09355","repositories_listed":5,"syntology":{"n":27,"n_ran":9,"n_unverified":18,"n_pointer_only":27}},{"url":"/paper/bge-m3-embedding-multi-lingual-multi","title":"BGE M3-Embedding: Multi-Lingual, Multi-Functionality, Multi-Granularity Text Embeddings Through Self-Knowledge Distillation","date":"2024-02-05","arxiv_id":"2402.03216","repositories_listed":4,"syntology":{"n":14,"n_ran":3,"n_unverified":11,"n_pointer_only":0}},{"url":"/paper/event-stream-based-visual-object-tracking-a","title":"Event Stream-based Visual Object Tracking: A High-Resolution Benchmark Dataset and A Novel Baseline","date":"2023-09-26","arxiv_id":"2309.14611","repositories_listed":4,"syntology":{"n":9,"n_ran":9,"n_unverified":0,"n_pointer_only":9}},{"url":"/paper/orca-progressive-learning-from-complex","title":"Orca: Progressive Learning from Complex Explanation Traces of GPT-4","date":"2023-06-05","arxiv_id":"2306.02707","repositories_listed":4,"syntology":null},{"url":"/paper/decoupled-kullback-leibler-divergence-loss","title":"Decoupled Kullback-Leibler Divergence Loss","date":"2023-05-23","arxiv_id":"2305.13948","repositories_listed":4,"syntology":{"n":11,"n_ran":7,"n_unverified":4,"n_pointer_only":4}},{"url":"/paper/a-closer-look-at-the-training-dynamics-of","title":"Understanding the Role of the Projector in Knowledge Distillation","date":"2023-03-20","arxiv_id":"2303.11098","repositories_listed":4,"syntology":null},{"url":"/paper/autoencoders-as-cross-modal-teachers-can","title":"Autoencoders as Cross-Modal Teachers: Can Pretrained 2D Image Transformers Help 3D Representation Learning?","date":"2022-12-16","arxiv_id":"2212.08320","repositories_listed":4,"syntology":null},{"url":"/paper/sa-mlp-distilling-graph-knowledge-from-gnns","title":"SA-MLP: Distilling Graph Knowledge from GNNs into Structure-Aware MLP","date":"2022-10-18","arxiv_id":"2210.09609","repositories_listed":4,"syntology":{"n":5,"n_ran":0,"n_unverified":5,"n_pointer_only":0}},{"url":"/paper/prodiff-progressive-fast-diffusion-model-for","title":"ProDiff: Progressive Fast Diffusion Model For High-Quality Text-to-Speech","date":"2022-07-13","arxiv_id":"2207.06389","repositories_listed":4,"syntology":{"n":6,"n_ran":2,"n_unverified":4,"n_pointer_only":0}},{"url":"/paper/structured-sparse-r-cnn-for-direct-scene","title":"Structured Sparse R-CNN for Direct Scene Graph Generation","date":"2021-06-21","arxiv_id":"2106.10815","repositories_listed":4,"syntology":{"n":1,"n_ran":1,"n_unverified":0,"n_pointer_only":1}},{"url":"/paper/data-free-knowledge-distillation-for-1","title":"Data-Free Knowledge Distillation for Heterogeneous Federated Learning","date":"2021-05-20","arxiv_id":"2105.10056","repositories_listed":4,"syntology":null}],"syntology_records":22,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-09-24T18:15:14+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"}},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"}}