{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/structured-knowledge-distillation-for","title":"Structured Knowledge Distillation for Dense Prediction","arxiv_id":"1903.04197","date":"2019-03-11","proceeding":"CVPR 2019 6","authors":["Yifan Liu","Changyong Shun","Jingdong Wang","Chunhua Shen"],"abstract":"In this work, we consider transferring the structure information from large networks to compact ones for dense prediction tasks in computer vision. Previous knowledge distillation strategies used for dense prediction tasks often directly borrow the distillation scheme for image classification and perform knowledge distillation for each pixel separately, leading to sub-optimal performance. Here we propose to distill structured knowledge from large networks to compact networks, taking into account the fact that dense prediction is a structured prediction problem. Specifically, we study two structured distillation schemes: i) pair-wise distillation that distills the pair-wise similarities by building a static graph; and ii) holistic distillation that uses adversarial training to distill holistic knowledge. The effectiveness of our knowledge distillation approaches is demonstrated by experiments on three dense prediction tasks: semantic segmentation, depth estimation and object detection. Code is available at: https://git.io/StructKD","url_abs":"https://arxiv.org/abs/1903.04197v7","url_pdf":"https://arxiv.org/pdf/1903.04197v7.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"structured-knowledge-distillation-for","repo_url":"https://github.com/irfanICMLL/structure_knowledge_distillation","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"BSD-2-Clause"}}],"tasks":[{"task_slug":"depth-estimation","task_name":"Depth Estimation"},{"task_slug":"classification","task_name":"General Classification"},{"task_slug":"image-classification","task_name":"Image Classification"},{"task_slug":"knowledge-distillation","task_name":"Knowledge Distillation"},{"task_slug":"object-detection","task_name":"Object Detection"},{"task_slug":"prediction","task_name":"Prediction"},{"task_slug":"scene-parsing","task_name":"Scene Parsing"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"},{"task_slug":"structured-prediction","task_name":"Structured Prediction"},{"task_slug":"image-classification","task_name":"image-classification"},{"task_slug":"object-detection-1","task_name":"object-detection"}],"methods":[{"method_slug":"knowledge-distillation","method_name":"Knowledge Distillation"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1903.04197","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1903.04197"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/irfanICMLL/structure_knowledge_distillation","reach":{"status":"ok","spdx":"BSD-2-Clause"}}],"summary":{"ran_draft_wrong":2,"ran_honours":1,"ran_fixture":1,"unverified":1},"by_repo_kind":{"official":{"samples":5,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"fac5364e2f53c6db","entry":"conv3x3","repo":"irfanICMLL/structure_knowledge_distillation","repo_kind":"official","path":"networks/pspnet_combine.py","file_url":"https://github.com/irfanICMLL/structure_knowledge_distillation/blob/HEAD/networks/pspnet_combine.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"fac5364e2f53c6db"}},{"code_sha256_prefix":"26664d79586aad5a","entry":"get_palette","repo":"irfanICMLL/structure_knowledge_distillation","repo_kind":"official","path":"networks/evaluate.py","file_url":"https://github.com/irfanICMLL/structure_knowledge_distillation/blob/HEAD/networks/evaluate.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"26664d79586aad5a"}},{"code_sha256_prefix":"bedff51745d2cf84","entry":"l2normalize","repo":"irfanICMLL/structure_knowledge_distillation","repo_kind":"official","path":"networks/spectral.py","file_url":"https://github.com/irfanICMLL/structure_knowledge_distillation/blob/HEAD/networks/spectral.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"bedff51745d2cf84"}},{"code_sha256_prefix":"6ec02fe7ffee8530","entry":"pad_image","repo":"irfanICMLL/structure_knowledge_distillation","repo_kind":"official","path":"networks/evaluate.py","file_url":"https://github.com/irfanICMLL/structure_knowledge_distillation/blob/HEAD/networks/evaluate.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"6ec02fe7ffee8530"}},{"code_sha256_prefix":"81ce123b96c24bd1","entry":"id2trainId","repo":"irfanICMLL/structure_knowledge_distillation","repo_kind":"official","path":"networks/evaluate.py","file_url":"https://github.com/irfanICMLL/structure_knowledge_distillation/blob/HEAD/networks/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"81ce123b96c24bd1"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}