{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/ensemble-knowledge-distillation-for-learning","title":"Ensemble Knowledge Distillation for Learning Improved and Efficient Networks","arxiv_id":"1909.08097","date":"2019-09-17","proceeding":null,"authors":["Umar Asif","Jianbin Tang","Stefan Harrer"],"abstract":"Ensemble models comprising of deep Convolutional Neural Networks (CNN) have shown significant improvements in model generalization but at the cost of large computation and memory requirements. In this paper, we present a framework for learning compact CNN models with improved classification performance and model generalization. For this, we propose a CNN architecture of a compact student model with parallel branches which are trained using ground truth labels and information from high capacity teacher networks in an ensemble learning fashion. Our framework provides two main benefits: i) Distilling knowledge from different teachers into the student network promotes heterogeneity in feature learning at different branches of the student network and enables the network to learn diverse solutions to the target problem. ii) Coupling the branches of the student network through ensembling encourages collaboration and improves the quality of the final predictions by reducing variance in the network outputs. Experiments on the well established CIFAR-10 and CIFAR-100 datasets show that our Ensemble Knowledge Distillation (EKD) improves classification accuracy and model generalization especially in situations with limited training data. Experiments also show that our EKD based compact networks outperform in terms of mean accuracy on the test datasets compared to state-of-the-art knowledge distillation based methods.","url_abs":"https://arxiv.org/abs/1909.08097v3","url_pdf":"https://arxiv.org/pdf/1909.08097v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"ensemble-knowledge-distillation-for-learning","repo_url":"https://github.com/softsys4ai/neural-distiller","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"ensemble-knowledge-distillation-for-learning","repo_url":"https://github.com/Adlik/model_optimizer","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"ensemble-learning","task_name":"Ensemble Learning"},{"task_slug":"classification","task_name":"General Classification"},{"task_slug":"knowledge-distillation","task_name":"Knowledge Distillation"}],"methods":[{"method_slug":"knowledge-distillation","method_name":"Knowledge Distillation"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/knowledge-distillation-on-imagenet","task":"Knowledge Distillation","dataset":"ImageNet","model":"ADLIK-MO-P25(T:SeNet154, ResNet152b  S:ResNet-50-prune25%)","rank_in_archive_order":17,"of":52,"metrics":{"CRD training setting":"✘","Top-1 accuracy %":"78.79","model size":"56.9M"},"uses_additional_data":false},{"leaderboard":"/sota/knowledge-distillation-on-imagenet","task":"Knowledge Distillation","dataset":"ImageNet","model":"ADLIK-MO-P375(T:SeNet154, ResNet152b  S:ResNet-50-prune37.5)","rank_in_archive_order":18,"of":52,"metrics":{"CRD training setting":"✘","Top-1 accuracy %":"78.07","model size":"40.5M"},"uses_additional_data":false},{"leaderboard":"/sota/knowledge-distillation-on-imagenet","task":"Knowledge Distillation","dataset":"ImageNet","model":"ADLIK-MO-P50(T:SeNet154, ResNet152b  S:ResNet-50-half)","rank_in_archive_order":24,"of":52,"metrics":{"CRD training setting":"✘","Top-1 accuracy %":"76.376","model size":"27M"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1909.08097","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1909.08097"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/softsys4ai/neural-distiller","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Adlik/model_optimizer","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"unverified":6},"by_repo_kind":{"listed":{"samples":6,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"5cb9054e3353acdb","entry":"accuracy","repo":"Adlik/model_optimizer","repo_kind":"listed","path":"src/model_optimizer/core/utils.py","file_url":"https://github.com/Adlik/model_optimizer/blob/HEAD/src/model_optimizer/core/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5cb9054e3353acdb"}},{"code_sha256_prefix":"3b1c6af3a7240f56","entry":"check_input_data","repo":"Adlik/model_optimizer","repo_kind":"listed","path":"src/model_optimizer/quantizer/deploy_fx_with_backend.py","file_url":"https://github.com/Adlik/model_optimizer/blob/HEAD/src/model_optimizer/quantizer/deploy_fx_with_backend.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3b1c6af3a7240f56"}},{"code_sha256_prefix":"184af36e7222c153","entry":"fake_quantize_per_channel_affine","repo":"Adlik/model_optimizer","repo_kind":"listed","path":"src/model_optimizer/quantizer/fake_quantize.py","file_url":"https://github.com/Adlik/model_optimizer/blob/HEAD/src/model_optimizer/quantizer/fake_quantize.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"184af36e7222c153"}},{"code_sha256_prefix":"cdfa1049b88fc86a","entry":"fake_quantize_per_tensor_affine","repo":"Adlik/model_optimizer","repo_kind":"listed","path":"src/model_optimizer/quantizer/fake_quantize.py","file_url":"https://github.com/Adlik/model_optimizer/blob/HEAD/src/model_optimizer/quantizer/fake_quantize.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cdfa1049b88fc86a"}},{"code_sha256_prefix":"a8e0390827e00583","entry":"fused_moving_avg_obs_fake_quant","repo":"Adlik/model_optimizer","repo_kind":"listed","path":"src/model_optimizer/quantizer/fake_quantize.py","file_url":"https://github.com/Adlik/model_optimizer/blob/HEAD/src/model_optimizer/quantizer/fake_quantize.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a8e0390827e00583"}},{"code_sha256_prefix":"ca2d9e1876635ca0","entry":"register_parser","repo":"Adlik/model_optimizer","repo_kind":"listed","path":"src/model_optimizer/pruners/structure_pruning.py","file_url":"https://github.com/Adlik/model_optimizer/blob/HEAD/src/model_optimizer/pruners/structure_pruning.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ca2d9e1876635ca0"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}