{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2601-14603","title":"Variance-Adaptive Muon: Accelerating LLM Pretraining with NSR-Modulated and Variance-Scaled Momentum","arxiv_id":"2601.14603","date":"2026-01-21","proceeding":null,"authors":["Jingru Li","Yibo Fan","Huan Li"],"abstract":"Large Language Models (LLMs) achieve competitive performance across diverse natural language processing (NLP) tasks, yet pretraining is computationally demanding, making optimizer efficiency an important practical consideration. Muon accelerates LLM pretraining via orthogonal momentum updates that serve as a matrix analogue of the element-wise sign operator. Motivated by the recent perspective that Adam is a variance-adaptive sign update algorithm, we propose two variants of Muon, Muon-NSR and Muon-VS, which apply variance-adaptive normalization to momentum before orthogonalization. Muon-NSR applies noise-to-signal ratio (NSR) modulation, while Muon-VS performs variance-based scaling without introducing additional hyperparameters. Experiments on GPT-2 and LLaMA pretraining demonstrate that our proposed methods accelerate convergence and consistently achieve lower validation loss than both competitive, well-tuned AdamW and Muon baselines. For example, on the LLaMA-1.2B model, Muon-NSR and Muon-VS reduce the iterations required to reach the target validation loss by $1.36\\times$ relative to the well-tuned Muon following the recent benchmark.","url_abs":"https://arxiv.org/abs/2601.14603","url_pdf":"https://arxiv.org/pdf/2601.14603","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2601.14603","atlas_url":"https://app.syntology.ai/?focus=2601.14603","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2601.14603"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/jingru-lee/Variance-Adaptive-Muon","reach":null}],"summary":{"ran":3,"unverified":4},"by_repo_kind":{"found_in_text":{"samples":7,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"9f67a4fea34dc5e4","entry":"adjust_lr_wd_for_muon","repo":"jingru-lee/Variance-Adaptive-Muon","repo_kind":"found_in_text","path":"Suite_A/src/optim/muon.py","file_url":"https://github.com/jingru-lee/Variance-Adaptive-Muon/blob/HEAD/Suite_A/src/optim/muon.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9f67a4fea34dc5e4"}},{"code_sha256_prefix":"ccaf2bdc7c8eeafa","entry":"normalize_range","repo":"jingru-lee/Variance-Adaptive-Muon","repo_kind":"found_in_text","path":"Suite_A/src/optim/muon.py","file_url":"https://github.com/jingru-lee/Variance-Adaptive-Muon/blob/HEAD/Suite_A/src/optim/muon.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ccaf2bdc7c8eeafa"}},{"code_sha256_prefix":"04049769e4eed44a","entry":"zeropower_via_newtonschulz5","repo":"jingru-lee/Variance-Adaptive-Muon","repo_kind":"found_in_text","path":"Suite_A/src/optim/muon.py","file_url":"https://github.com/jingru-lee/Variance-Adaptive-Muon/blob/HEAD/Suite_A/src/optim/muon.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"04049769e4eed44a"}},{"code_sha256_prefix":"335254babb23e8b3","entry":"normalize_range","repo":"jingru-lee/Variance-Adaptive-Muon","repo_kind":"found_in_text","path":"Suite_A/src/optim/muon_vs.py","file_url":"https://github.com/jingru-lee/Variance-Adaptive-Muon/blob/HEAD/Suite_A/src/optim/muon_vs.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"335254babb23e8b3"}},{"code_sha256_prefix":"bf190c2d81f45aa9","entry":"zeropower_via_newtonschulz5","repo":"jingru-lee/Variance-Adaptive-Muon","repo_kind":"found_in_text","path":"Suite_B/muon.py","file_url":"https://github.com/jingru-lee/Variance-Adaptive-Muon/blob/HEAD/Suite_B/muon.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bf190c2d81f45aa9"}},{"code_sha256_prefix":"901cfa8d98a949c5","entry":"zeropower_via_newtonschulz5","repo":"jingru-lee/Variance-Adaptive-Muon","repo_kind":"found_in_text","path":"Suite_B/muon_nsr.py","file_url":"https://github.com/jingru-lee/Variance-Adaptive-Muon/blob/HEAD/Suite_B/muon_nsr.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"901cfa8d98a949c5"}},{"code_sha256_prefix":"72f7d6d5b8308b8e","entry":"zeropower_via_newtonschulz5","repo":"jingru-lee/Variance-Adaptive-Muon","repo_kind":"found_in_text","path":"Suite_A/src/optim/muon_vs.py","file_url":"https://github.com/jingru-lee/Variance-Adaptive-Muon/blob/HEAD/Suite_A/src/optim/muon_vs.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"72f7d6d5b8308b8e"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.LG","source":"arxiv_2026.jsonl"},"syntology_extracted_results":null}