{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/tensor-programs-v-tuning-large-neural","title":"Tensor Programs V: Tuning Large Neural Networks via Zero-Shot Hyperparameter Transfer","arxiv_id":"2203.03466","date":"2022-03-07","proceeding":null,"authors":["Greg Yang","Edward J. Hu","Igor Babuschkin","Szymon Sidor","Xiaodong Liu","David Farhi","Nick Ryder","Jakub Pachocki","Weizhu Chen","Jianfeng Gao"],"abstract":"Hyperparameter (HP) tuning in deep learning is an expensive process, prohibitively so for neural networks (NNs) with billions of parameters. We show that, in the recently discovered Maximal Update Parametrization (muP), many optimal HPs remain stable even as model size changes. This leads to a new HP tuning paradigm we call muTransfer: parametrize the target model in muP, tune the HP indirectly on a smaller model, and zero-shot transfer them to the full-sized model, i.e., without directly tuning the latter at all. We verify muTransfer on Transformer and ResNet. For example, 1) by transferring pretraining HPs from a model of 13M parameters, we outperform published numbers of BERT-large (350M parameters), with a total tuning cost equivalent to pretraining BERT-large once; 2) by transferring from 40M parameters, we outperform published numbers of the 6.7B GPT-3 model, with tuning cost only 7% of total pretraining cost. A Pytorch implementation of our technique can be found at github.com/microsoft/mup and installable via `pip install mup`.","url_abs":"https://arxiv.org/abs/2203.03466v2","url_pdf":"https://arxiv.org/pdf/2203.03466v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"tensor-programs-v-tuning-large-neural","repo_url":"https://github.com/microsoft/mup","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"tensor-programs-v-tuning-large-neural","repo_url":"https://github.com/ML-GSAI/Scaling-Diffusion-Transformers-muP","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"tensor-programs-v-tuning-large-neural","repo_url":"https://github.com/clankur/muGPT","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":{"status":"ok","spdx":"BSD-3-Clause"}},{"paper_slug":"tensor-programs-v-tuning-large-neural","repo_url":"https://github.com/eleutherai/nanogpt-mup","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"tensor-programs-v-tuning-large-neural","repo_url":"https://github.com/lucaslingle/mu_transformer","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":null},{"paper_slug":"tensor-programs-v-tuning-large-neural","repo_url":"https://github.com/mu-transformer-authors/mu_transformer","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":null},{"paper_slug":"tensor-programs-v-tuning-large-neural","repo_url":"https://github.com/vita-group/principled_scaling_lr_init","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[],"methods":[{"method_slug":"1x1-convolution","method_name":"1x1 Convolution"},{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"average-pooling","method_name":"Average Pooling"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"batch-normalization","method_name":"Batch Normalization"},{"method_slug":"bottleneck-residual-block","method_name":"Bottleneck Residual Block"},{"method_slug":"convolution","method_name":"Convolution"},{"method_slug":"cosine-annealing","method_name":"Cosine Annealing"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt-3","method_name":"GPT-3"},{"method_slug":"global-average-pooling","method_name":"Global Average Pooling"},{"method_slug":"kaiming-initialization","method_name":"Kaiming Initialization"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-cosine-annealing","method_name":"Linear Warmup With Cosine Annealing"},{"method_slug":"max-pooling","method_name":"Max Pooling"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"relu","method_name":"ReLU"},{"method_slug":"residual-block","method_name":"Residual Block"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"},{"method_slug":"weight-decay","method_name":"Weight Decay"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2203.03466","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.03466"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/microsoft/mup","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/lucaslingle/mu_transformer","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/clankur/muGPT","reach":{"status":"ok","spdx":"BSD-3-Clause"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/vita-group/principled_scaling_lr_init","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/eleutherai/nanogpt-mup","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ML-GSAI/Scaling-Diffusion-Transformers-muP","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mu-transformer-authors/mu_transformer","reach":null}],"summary":{"ran":1,"ran_fixture":1,"ran_draft_wrong":1,"unverified":2},"by_repo_kind":{"official":{"samples":1,"ran":1,"repositories":1},"listed":{"samples":4,"ran":2,"repositories":3}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"cc51a0499aef9a80","entry":"TransformerConfig","repo":"mu-transformer-authors/mu_transformer","repo_kind":"listed","path":"mu_transformer/jax_impl/model.py","file_url":"https://github.com/mu-transformer-authors/mu_transformer/blob/HEAD/mu_transformer/jax_impl/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cc51a0499aef9a80"}},{"code_sha256_prefix":"0c4c2c6a29fb9431","entry":"constant_std_init_","repo":"microsoft/mup","repo_kind":"official","path":"mup/init.py","file_url":"https://github.com/microsoft/mup/blob/HEAD/mup/init.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0c4c2c6a29fb9431"}},{"code_sha256_prefix":"cc4b5a2dc1f8c1e4","entry":"critic_init_","repo":"vita-group/principled_scaling_lr_init","repo_kind":"listed","path":"models/utils.py","file_url":"https://github.com/vita-group/principled_scaling_lr_init/blob/HEAD/models/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cc4b5a2dc1f8c1e4"}},{"code_sha256_prefix":"c8add3cbe90a58e8","entry":"_init","repo":"vita-group/principled_scaling_lr_init","repo_kind":"listed","path":"models/utils.py","file_url":"https://github.com/vita-group/principled_scaling_lr_init/blob/HEAD/models/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c8add3cbe90a58e8"}},{"code_sha256_prefix":"86cf6316062ec680","entry":"get_rel_mup_scaling","repo":"lucaslingle/mu_transformer","repo_kind":"listed","path":"mu_transformer/jax_impl/launch.py","file_url":"https://github.com/lucaslingle/mu_transformer/blob/HEAD/mu_transformer/jax_impl/launch.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"86cf6316062ec680"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}