{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/distilling-knowledge-via-knowledge-review","title":"Distilling Knowledge via Knowledge Review","arxiv_id":"2104.09044","date":"2021-04-19","proceeding":"CVPR 2021 1","authors":["Pengguang Chen","Shu Liu","Hengshuang Zhao","Jiaya Jia"],"abstract":"Knowledge distillation transfers knowledge from the teacher network to the student one, with the goal of greatly improving the performance of the student network. Previous methods mostly focus on proposing feature transformation and loss functions between the same level's features to improve the effectiveness. We differently study the factor of connection path cross levels between teacher and student networks, and reveal its great importance. For the first time in knowledge distillation, cross-stage connection paths are proposed. Our new review mechanism is effective and structurally simple. Our finally designed nested and compact framework requires negligible computation overhead, and outperforms other methods on a variety of tasks. We apply our method to classification, object detection, and instance segmentation tasks. All of them witness significant student network performance improvement. Code is available at https://github.com/Jia-Research-Lab/ReviewKD","url_abs":"https://arxiv.org/abs/2104.09044v1","url_pdf":"https://arxiv.org/pdf/2104.09044v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"distilling-knowledge-via-knowledge-review","repo_url":"https://github.com/Jia-Research-Lab/ReviewKD","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"distilling-knowledge-via-knowledge-review","repo_url":"https://github.com/DevPranjal/reproduction-review-kd","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"distilling-knowledge-via-knowledge-review","repo_url":"https://github.com/ZJCV/KnowledgeReview","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"distilling-knowledge-via-knowledge-review","repo_url":"https://github.com/dvlab-research/reviewkd","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"distilling-knowledge-via-knowledge-review","repo_url":"https://github.com/ruipingl/skr_pea","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"distilling-knowledge-via-knowledge-review","repo_url":"https://github.com/ruipingl/transkd","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"distilling-knowledge-via-knowledge-review","repo_url":"https://github.com/yoshitomo-matsubara/torchdistill","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"instance-segmentation","task_name":"Instance Segmentation"},{"task_slug":"knowledge-distillation","task_name":"Knowledge Distillation"},{"task_slug":"object-detection","task_name":"Object Detection"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"},{"task_slug":"object-detection-1","task_name":"object-detection"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/knowledge-distillation-on-cifar-100","task":"Knowledge Distillation","dataset":"CIFAR-100","model":"resnet8x4 (T: resnet32x4 S: resnet8x4)","rank_in_archive_order":14,"of":27,"metrics":{"Top-1 Accuracy (%)":"75.63"},"uses_additional_data":false},{"leaderboard":"/sota/knowledge-distillation-on-cifar-100","task":"Knowledge Distillation","dataset":"CIFAR-100","model":"vgg8 (T:vgg13 S:vgg8)","rank_in_archive_order":17,"of":27,"metrics":{"Top-1 Accuracy (%)":"74.84"},"uses_additional_data":false},{"leaderboard":"/sota/knowledge-distillation-on-imagenet","task":"Knowledge Distillation","dataset":"ImageNet","model":"Knowledge Review (T: ResNet-34  S:ResNet-18)","rank_in_archive_order":44,"of":52,"metrics":{"CRD training setting":"✓","Top-1 accuracy %":"71.61"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2104.09044","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.09044"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yoshitomo-matsubara/torchdistill","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/dvlab-research/reviewkd","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ZJCV/KnowledgeReview","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/DevPranjal/reproduction-review-kd","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ruipingl/transkd","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ruipingl/skr_pea","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Jia-Research-Lab/ReviewKD","reach":null}],"summary":{"ran":4,"ran_draft_wrong":1,"unverified":3},"by_repo_kind":{"official":{"samples":2,"ran":0,"repositories":1},"listed":{"samples":6,"ran":5,"repositories":4}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":5,"samples":[{"code_sha256_prefix":"6613345022d326fc","entry":"ABF","repo":"ruipingl/skr_pea","repo_kind":"listed","path":"train/ReviewKD.py","file_url":"https://github.com/ruipingl/skr_pea/blob/HEAD/train/ReviewKD.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6613345022d326fc"}},{"code_sha256_prefix":"407f14bd274a60dd","entry":"HierarchicalContextLoss","repo":"yoshitomo-matsubara/torchdistill","repo_kind":"listed","path":"torchdistill/losses/mid_level.py","file_url":"https://github.com/yoshitomo-matsubara/torchdistill/blob/HEAD/torchdistill/losses/mid_level.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"407f14bd274a60dd"}},{"code_sha256_prefix":"3bb723e9c090a1d0","entry":"RLF_for_Resnet","repo":"DevPranjal/reproduction-review-kd","repo_kind":"listed","path":"framework.py","file_url":"https://github.com/DevPranjal/reproduction-review-kd/blob/HEAD/framework.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3bb723e9c090a1d0"}},{"code_sha256_prefix":"65da0d1fda7c81fe","entry":"ReviewKD","repo":"ruipingl/skr_pea","repo_kind":"listed","path":"train/ReviewKD.py","file_url":"https://github.com/ruipingl/skr_pea/blob/HEAD/train/ReviewKD.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"65da0d1fda7c81fe"}},{"code_sha256_prefix":"3daf804e7ed5c814","entry":"register_mid_level_loss","repo":"yoshitomo-matsubara/torchdistill","repo_kind":"listed","path":"torchdistill/losses/mid_level.py","file_url":"https://github.com/yoshitomo-matsubara/torchdistill/blob/HEAD/torchdistill/losses/mid_level.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3daf804e7ed5c814"}},{"code_sha256_prefix":"8db7e32366ae454d","entry":"ABF","repo":"Jia-Research-Lab/ReviewKD","repo_kind":"official","path":"CIFAR-100/model/reviewkd.py","file_url":"https://github.com/Jia-Research-Lab/ReviewKD/blob/HEAD/CIFAR-100/model/reviewkd.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8db7e32366ae454d"}},{"code_sha256_prefix":"cc67bd1bd51e7d3b","entry":"ReviewKD","repo":"Jia-Research-Lab/ReviewKD","repo_kind":"official","path":"CIFAR-100/model/reviewkd.py","file_url":"https://github.com/Jia-Research-Lab/ReviewKD/blob/HEAD/CIFAR-100/model/reviewkd.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"cc67bd1bd51e7d3b"}},{"code_sha256_prefix":"7919e3e477475e60","entry":"hcl","repo":"ZJCV/KnowledgeReview","repo_kind":"listed","path":"rfd/criterion/rfd_loss.py","file_url":"https://github.com/ZJCV/KnowledgeReview/blob/HEAD/rfd/criterion/rfd_loss.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7919e3e477475e60"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}