{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/a-comprehensive-overhaul-of-feature","title":"A Comprehensive Overhaul of Feature Distillation","arxiv_id":"1904.01866","date":"2019-04-03","proceeding":"ICCV 2019 10","authors":["Byeongho Heo","Jeesoo Kim","Sangdoo Yun","Hyojin Park","Nojun Kwak","Jin Young Choi"],"abstract":"We investigate the design aspects of feature distillation methods achieving network compression and propose a novel feature distillation method in which the distillation loss is designed to make a synergy among various aspects: teacher transform, student transform, distillation feature position and distance function. Our proposed distillation loss includes a feature transform with a newly designed margin ReLU, a new distillation feature position, and a partial L2 distance function to skip redundant information giving adverse effects to the compression of student. In ImageNet, our proposed method achieves 21.65% of top-1 error with ResNet50, which outperforms the performance of the teacher network, ResNet152. Our proposed method is evaluated on various tasks such as image classification, object detection and semantic segmentation and achieves a significant performance improvement in all tasks. The code is available at https://sites.google.com/view/byeongho-heo/overhaul","url_abs":"https://arxiv.org/abs/1904.01866v2","url_pdf":"https://arxiv.org/pdf/1904.01866v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"a-comprehensive-overhaul-of-feature","repo_url":"https://github.com/clovaai/overhaul-distillation","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"a-comprehensive-overhaul-of-feature","repo_url":"https://github.com/kaung-htet-myat/feature-distillation-tf","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}}],"tasks":[{"task_slug":"classification","task_name":"General Classification"},{"task_slug":"image-classification","task_name":"Image Classification"},{"task_slug":"knowledge-distillation","task_name":"Knowledge Distillation"},{"task_slug":"object-detection","task_name":"Object Detection"},{"task_slug":null,"task_name":"Position"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"},{"task_slug":"image-classification","task_name":"image-classification"},{"task_slug":"object-detection-1","task_name":"object-detection"}],"methods":[{"method_slug":"1-bit-adam","method_name":"1-bit Adam"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"margin-relu","method_name":"Margin ReLU"},{"method_slug":"relu","method_name":"ReLU"}],"datasets_introduced":[],"methods_introduced":[{"slug":"margin-relu","name":"Margin ReLU","full_name":"Margin Rectified Linear Unit"}],"results":[{"leaderboard":"/sota/knowledge-distillation-on-imagenet","task":"Knowledge Distillation","dataset":"ImageNet","model":"Overhual (T: ResNet-34 S:ResNet-18)","rank_in_archive_order":49,"of":52,"metrics":{"Top-1 accuracy %":"70.81"},"uses_additional_data":false}],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=1904.01866","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1904.01866"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/clovaai/overhaul-distillation","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kaung-htet-myat/feature-distillation-tf","reach":{"status":"ok"}}],"summary":{"ran_draft_wrong":1,"unverified":5},"by_repo_kind":{"official":{"samples":6,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"fac5364e2f53c6db","entry":"conv3x3","repo":"clovaai/overhaul-distillation","repo_kind":"official","path":"CIFAR-100/models/PyramidNet.py","file_url":"https://github.com/clovaai/overhaul-distillation/blob/HEAD/CIFAR-100/models/PyramidNet.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fac5364e2f53c6db"}},{"code_sha256_prefix":"8f46ca5ce495a9c0","entry":"build_feature_connector","repo":"clovaai/overhaul-distillation","repo_kind":"official","path":"ImageNet/distiller.py","file_url":"https://github.com/clovaai/overhaul-distillation/blob/HEAD/ImageNet/distiller.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8f46ca5ce495a9c0"}},{"code_sha256_prefix":"986a20dced4a5241","entry":"distillation_loss","repo":"clovaai/overhaul-distillation","repo_kind":"official","path":"ImageNet/distiller.py","file_url":"https://github.com/clovaai/overhaul-distillation/blob/HEAD/ImageNet/distiller.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"986a20dced4a5241"}},{"code_sha256_prefix":"f49b50f56067512d","entry":"get_margin_from_BN","repo":"clovaai/overhaul-distillation","repo_kind":"official","path":"ImageNet/distiller.py","file_url":"https://github.com/clovaai/overhaul-distillation/blob/HEAD/ImageNet/distiller.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f49b50f56067512d"}},{"code_sha256_prefix":"d17135c845d34e76","entry":"resnet18","repo":"clovaai/overhaul-distillation","repo_kind":"official","path":"ImageNet/models/ResNet.py","file_url":"https://github.com/clovaai/overhaul-distillation/blob/HEAD/ImageNet/models/ResNet.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d17135c845d34e76"}},{"code_sha256_prefix":"3012931e32b696c4","entry":"resnet34","repo":"clovaai/overhaul-distillation","repo_kind":"official","path":"ImageNet/models/ResNet.py","file_url":"https://github.com/clovaai/overhaul-distillation/blob/HEAD/ImageNet/models/ResNet.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3012931e32b696c4"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}