{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/optimizing-millions-of-hyperparameters-by","title":"Optimizing Millions of Hyperparameters by Implicit Differentiation","arxiv_id":"1911.02590","date":"2019-11-06","proceeding":null,"authors":["Jonathan Lorraine","Paul Vicol","David Duvenaud"],"abstract":"We propose an algorithm for inexpensive gradient-based hyperparameter optimization that combines the implicit function theorem (IFT) with efficient inverse Hessian approximations. We present results about the relationship between the IFT and differentiating through optimization, motivating our algorithm. We use the proposed approach to train modern network architectures with millions of weights and millions of hyper-parameters. For example, we learn a data-augmentation network - where every weight is a hyperparameter tuned for validation performance - outputting augmented training examples. Jointly tuning weights and hyperparameters with our approach is only a few times more costly in memory and compute than standard training.","url_abs":"https://arxiv.org/abs/1911.02590v1","url_pdf":"https://arxiv.org/pdf/1911.02590v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"optimizing-millions-of-hyperparameters-by","repo_url":"https://github.com/Guang000/Awesome-Dataset-Distillation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"optimizing-millions-of-hyperparameters-by","repo_url":"https://github.com/Tarzanagh/FedNest","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"optimizing-millions-of-hyperparameters-by","repo_url":"https://github.com/mc-nya/fednest","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"optimizing-millions-of-hyperparameters-by","repo_url":"https://github.com/mocchi-tam/pytorch-HO-implicit-diff","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"optimizing-millions-of-hyperparameters-by","repo_url":"https://github.com/ucr-optml/FedNest","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"optimizing-millions-of-hyperparameters-by","repo_url":"https://github.com/ucr-optml/fedmsa","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"optimizing-millions-of-hyperparameters-by","repo_url":"https://github.com/AvivNavon/AuxiLearn","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":null},{"paper_slug":"optimizing-millions-of-hyperparameters-by","repo_url":"https://github.com/MaximeVandegar/Papers-in-100-Lines-of-Code/tree/main/Optimizing_Millions_of_Hyperparameters_by_Implicit_Differentiation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":null},{"paper_slug":"optimizing-millions-of-hyperparameters-by","repo_url":"https://github.com/mmcdermott/pytorch_implicit_gradients","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"data-augmentation","task_name":"Data Augmentation"},{"task_slug":"hyperparameter-optimization","task_name":"Hyperparameter Optimization"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1911.02590","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1911.02590"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/AvivNavon/AuxiLearn","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ucr-optml/FedNest","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Guang000/Awesome-Dataset-Distillation","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mocchi-tam/pytorch-HO-implicit-diff","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mc-nya/fednest","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ucr-optml/fedmsa","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/MaximeVandegar/Papers-in-100-Lines-of-Code/tree/main/Optimizing_Millions_of_Hyperparameters_by_Implicit_Differentiation","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Tarzanagh/FedNest","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mmcdermott/pytorch_implicit_gradients","reach":{"status":"ok"}}],"summary":{"ran_honours":1,"ran_fixture":1,"ran_violates":1,"unverified":1},"by_repo_kind":{"listed":{"samples":4,"ran":3,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"739afdca8b2d673a","entry":"gather_flat_grad","repo":"ucr-optml/fedmsa","repo_kind":"listed","path":"core/function.py","file_url":"https://github.com/ucr-optml/fedmsa/blob/HEAD/core/function.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"739afdca8b2d673a"}},{"code_sha256_prefix":"085767456c708d9d","entry":"neumann_hyperstep_preconditioner","repo":"ucr-optml/fedmsa","repo_kind":"listed","path":"core/function.py","file_url":"https://github.com/ucr-optml/fedmsa/blob/HEAD/core/function.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"085767456c708d9d"}},{"code_sha256_prefix":"97aa6390755cc0a7","entry":"smooth","repo":"ucr-optml/FedNest","repo_kind":"listed","path":"reproduce/fig2.py","file_url":"https://github.com/ucr-optml/FedNest/blob/HEAD/reproduce/fig2.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"97aa6390755cc0a7"}},{"code_sha256_prefix":"2b5f9d39a738d9b9","entry":"loss_adjust_cross_entropy","repo":"ucr-optml/fedmsa","repo_kind":"listed","path":"core/function.py","file_url":"https://github.com/ucr-optml/fedmsa/blob/HEAD/core/function.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2b5f9d39a738d9b9"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}