{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/towards-a-theory-of-model-distillation","title":"Towards a theory of model distillation","arxiv_id":"2403.09053","date":"2024-03-14","proceeding":null,"authors":["Enric Boix-Adsera"],"abstract":"Distillation is the task of replacing a complicated machine learning model with a simpler model that approximates the original [BCNM06,HVD15]. Despite many practical applications, basic questions about the extent to which models can be distilled, and the runtime and amount of data needed to distill, remain largely open. To study these questions, we initiate a general theory of distillation, defining PAC-distillation in an analogous way to PAC-learning [Val84]. As applications of this theory: (1) we propose new algorithms to extract the knowledge stored in the trained weights of neural networks -- we show how to efficiently distill neural networks into succinct, explicit decision tree representations when possible by using the ``linear representation hypothesis''; and (2) we prove that distillation can be much cheaper than learning from scratch, and make progress on characterizing its complexity.","url_abs":"https://arxiv.org/abs/2403.09053v2","url_pdf":"https://arxiv.org/pdf/2403.09053v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"towards-a-theory-of-model-distillation","repo_url":"https://github.com/eboix/theory-of-model-distillation","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"pac-learning","task_name":"PAC learning"},{"task_slug":"model","task_name":"model"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2403.09053","atlas_url":"https://app.syntology.ai/?focus=2403.09053","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.09053"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/eboix/theory-of-model-distillation","reach":{"status":"ok"}}],"summary":{"ran":8},"by_repo_kind":{"official":{"samples":8,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":8,"samples":[{"code_sha256_prefix":"b62beacb52e18332","entry":"batch_linear_probe_adam","repo":"eboix/theory-of-model-distillation","repo_kind":"official","path":"probing_utils.py","file_url":"https://github.com/eboix/theory-of-model-distillation/blob/HEAD/probing_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b62beacb52e18332"}},{"code_sha256_prefix":"20fe419b3cd8dfbf","entry":"batch_logistic_probe_adam","repo":"eboix/theory-of-model-distillation","repo_kind":"official","path":"probing_utils.py","file_url":"https://github.com/eboix/theory-of-model-distillation/blob/HEAD/probing_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"20fe419b3cd8dfbf"}},{"code_sha256_prefix":"01704e6e4419e521","entry":"comp_and","repo":"eboix/theory-of-model-distillation","repo_kind":"official","path":"decision_tree_utils.py","file_url":"https://github.com/eboix/theory-of-model-distillation/blob/HEAD/decision_tree_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"01704e6e4419e521"}},{"code_sha256_prefix":"55004cdca28a206f","entry":"comp_and_list","repo":"eboix/theory-of-model-distillation","repo_kind":"official","path":"decision_tree_utils.py","file_url":"https://github.com/eboix/theory-of-model-distillation/blob/HEAD/decision_tree_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"55004cdca28a206f"}},{"code_sha256_prefix":"fcae99f48ee56066","entry":"get_random_data_unif_binary","repo":"eboix/theory-of-model-distillation","repo_kind":"official","path":"train_utils.py","file_url":"https://github.com/eboix/theory-of-model-distillation/blob/HEAD/train_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fcae99f48ee56066"}},{"code_sha256_prefix":"39d9c79e201d3259","entry":"literals_to_tup","repo":"eboix/theory-of-model-distillation","repo_kind":"official","path":"decision_tree_utils.py","file_url":"https://github.com/eboix/theory-of-model-distillation/blob/HEAD/decision_tree_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"39d9c79e201d3259"}},{"code_sha256_prefix":"59983924736c5a8a","entry":"test_class","repo":"eboix/theory-of-model-distillation","repo_kind":"official","path":"train_utils.py","file_url":"https://github.com/eboix/theory-of-model-distillation/blob/HEAD/train_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"59983924736c5a8a"}},{"code_sha256_prefix":"51b5eb19649eb81d","entry":"validation_split","repo":"eboix/theory-of-model-distillation","repo_kind":"official","path":"train_utils.py","file_url":"https://github.com/eboix/theory-of-model-distillation/blob/HEAD/train_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"51b5eb19649eb81d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}