{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/simreg-regression-as-a-simple-yet-effective","title":"SimReg: Regression as a Simple Yet Effective Tool for Self-supervised Knowledge Distillation","arxiv_id":"2201.05131","date":"2022-01-13","proceeding":null,"authors":["K L Navaneet","Soroush Abbasi Koohpayegani","Ajinkya Tejankar","Hamed Pirsiavash"],"abstract":"Feature regression is a simple way to distill large neural network models to smaller ones. We show that with simple changes to the network architecture, regression can outperform more complex state-of-the-art approaches for knowledge distillation from self-supervised models. Surprisingly, the addition of a multi-layer perceptron head to the CNN backbone is beneficial even if used only during distillation and discarded in the downstream task. Deeper non-linear projections can thus be used to accurately mimic the teacher without changing inference architecture and time. Moreover, we utilize independent projection heads to simultaneously distill multiple teacher networks. We also find that using the same weakly augmented image as input for both teacher and student networks aids distillation. Experiments on ImageNet dataset demonstrate the efficacy of the proposed changes in various self-supervised distillation settings.","url_abs":"https://arxiv.org/abs/2201.05131v1","url_pdf":"https://arxiv.org/pdf/2201.05131v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"simreg-regression-as-a-simple-yet-effective","repo_url":"https://github.com/ucdvision/simreg","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"knowledge-distillation","task_name":"Knowledge Distillation"},{"task_slug":"regression-1","task_name":"regression"}],"methods":[{"method_slug":"knowledge-distillation","method_name":"Knowledge Distillation"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2201.05131","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2201.05131"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ucdvision/simreg","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":2,"ran_fixture":1,"unverified":11},"by_repo_kind":{"official":{"samples":14,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"fac5364e2f53c6db","entry":"conv3x3","repo":"ucdvision/simreg","repo_kind":"official","path":"models/resnet.py","file_url":"https://github.com/ucdvision/simreg/blob/HEAD/models/resnet.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fac5364e2f53c6db"}},{"code_sha256_prefix":"416b44e8cb9fc048","entry":"get_logger","repo":"ucdvision/simreg","repo_kind":"official","path":"tools.py","file_url":"https://github.com/ucdvision/simreg/blob/HEAD/tools.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"416b44e8cb9fc048"}},{"code_sha256_prefix":"e2e91e39d8368ced","entry":"get_shuffle_ids","repo":"ucdvision/simreg","repo_kind":"official","path":"tools.py","file_url":"https://github.com/ucdvision/simreg/blob/HEAD/tools.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e2e91e39d8368ced"}},{"code_sha256_prefix":"913d82af53065418","entry":"accuracy","repo":"ucdvision/simreg","repo_kind":"official","path":"tools.py","file_url":"https://github.com/ucdvision/simreg/blob/HEAD/tools.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"913d82af53065418"}},{"code_sha256_prefix":"cfc1c30393d20203","entry":"calc_padding_same","repo":"ucdvision/simreg","repo_kind":"official","path":"models/resnet_byol.py","file_url":"https://github.com/ucdvision/simreg/blob/HEAD/models/resnet_byol.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cfc1c30393d20203"}},{"code_sha256_prefix":"c4b3a9d234aede9e","entry":"conv3x3","repo":"ucdvision/simreg","repo_kind":"official","path":"models/resnet_byol.py","file_url":"https://github.com/ucdvision/simreg/blob/HEAD/models/resnet_byol.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c4b3a9d234aede9e"}},{"code_sha256_prefix":"ab7f2ccec8a2f284","entry":"get_linear_proj","repo":"ucdvision/simreg","repo_kind":"official","path":"eval_knn.py","file_url":"https://github.com/ucdvision/simreg/blob/HEAD/eval_knn.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ab7f2ccec8a2f284"}},{"code_sha256_prefix":"f197e60541e82784","entry":"get_linear_proj","repo":"ucdvision/simreg","repo_kind":"official","path":"mlp_arch.py","file_url":"https://github.com/ucdvision/simreg/blob/HEAD/mlp_arch.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f197e60541e82784"}},{"code_sha256_prefix":"f12879d61efb3165","entry":"get_mlp","repo":"ucdvision/simreg","repo_kind":"official","path":"eval_knn.py","file_url":"https://github.com/ucdvision/simreg/blob/HEAD/eval_knn.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f12879d61efb3165"}},{"code_sha256_prefix":"e26a5aa8fe2d9dd8","entry":"get_mlp","repo":"ucdvision/simreg","repo_kind":"official","path":"mlp_arch.py","file_url":"https://github.com/ucdvision/simreg/blob/HEAD/mlp_arch.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e26a5aa8fe2d9dd8"}},{"code_sha256_prefix":"7f32e37ef9afef4a","entry":"get_train_loader","repo":"ucdvision/simreg","repo_kind":"official","path":"dataloader.py","file_url":"https://github.com/ucdvision/simreg/blob/HEAD/dataloader.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7f32e37ef9afef4a"}},{"code_sha256_prefix":"b10ffd9ad7e3f637","entry":"pad_same","repo":"ucdvision/simreg","repo_kind":"official","path":"models/resnet_byol.py","file_url":"https://github.com/ucdvision/simreg/blob/HEAD/models/resnet_byol.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b10ffd9ad7e3f637"}},{"code_sha256_prefix":"53d621a63b9777b4","entry":"resnet18","repo":"ucdvision/simreg","repo_kind":"official","path":"models/resnet.py","file_url":"https://github.com/ucdvision/simreg/blob/HEAD/models/resnet.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"53d621a63b9777b4"}},{"code_sha256_prefix":"3983c46a5cacaa51","entry":"resnet50","repo":"ucdvision/simreg","repo_kind":"official","path":"models/resnet.py","file_url":"https://github.com/ucdvision/simreg/blob/HEAD/models/resnet.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3983c46a5cacaa51"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}