{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/on-the-adequacy-of-untuned-warmup-for","title":"On the adequacy of untuned warmup for adaptive optimization","arxiv_id":"1910.04209","date":"2019-10-09","proceeding":null,"authors":["Jerry Ma","Denis Yarats"],"abstract":"Adaptive optimization algorithms such as Adam are widely used in deep learning. The stability of such algorithms is often improved with a warmup schedule for the learning rate. Motivated by the difficulty of choosing and tuning warmup schedules, recent work proposes automatic variance rectification of Adam's adaptive learning rate, claiming that this rectified approach (\"RAdam\") surpasses the vanilla Adam algorithm and reduces the need for expensive tuning of Adam with warmup. In this work, we refute this analysis and provide an alternative explanation for the necessity of warmup based on the magnitude of the update term, which is of greater relevance to training stability. We then provide some \"rule-of-thumb\" warmup schedules, and we demonstrate that simple untuned warmup of Adam performs more-or-less identically to RAdam in typical practical settings. We conclude by suggesting that practitioners stick to linear warmup with Adam, with a sensible default being linear warmup over $2 / (1 - \\beta_2)$ training iterations.","url_abs":"https://arxiv.org/abs/1910.04209v3","url_pdf":"https://arxiv.org/pdf/1910.04209v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"on-the-adequacy-of-untuned-warmup-for","repo_url":"https://github.com/Tony-Y/pytorch_warmup","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"image-classification","task_name":"Image Classification"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"machine-translation","task_name":"Machine Translation"}],"methods":[{"method_slug":"1x1-convolution","method_name":"1x1 Convolution"},{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"average-pooling","method_name":"Average Pooling"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"batch-normalization","method_name":"Batch Normalization"},{"method_slug":"bottleneck-residual-block","method_name":"Bottleneck Residual Block"},{"method_slug":"convolution","method_name":"Convolution"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"global-average-pooling","method_name":"Global Average Pooling"},{"method_slug":"kaiming-initialization","method_name":"Kaiming Initialization"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup","method_name":"Linear Warmup"},{"method_slug":"max-pooling","method_name":"Max Pooling"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"radam","method_name":"RAdam"},{"method_slug":"relu","method_name":"ReLU"},{"method_slug":"residual-block","method_name":"Residual Block"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/image-classification-on-imagenet","task":"Image Classification","dataset":"ImageNet","model":"ResNet-50","rank_in_archive_order":1000,"of":1060,"metrics":{"Top 1 Accuracy":"72.1%"},"uses_additional_data":false},{"leaderboard":"/sota/language-modelling-on-wikitext-103","task":"Language Modelling","dataset":"WikiText-103","model":"Transformer  (Adaptive inputs)","rank_in_archive_order":86,"of":89,"metrics":{"Validation perplexity":"19.5"},"uses_additional_data":false},{"leaderboard":"/sota/machine-translation-on-wmt2016-english-german","task":"Machine Translation","dataset":"WMT2016 English-German","model":"Transformer","rank_in_archive_order":6,"of":12,"metrics":{"BLEU score":"26.7"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1910.04209","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1910.04209"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Tony-Y/pytorch_warmup","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":4},"by_repo_kind":{"listed":{"samples":4,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"4f02faccfbcfd169","entry":"get_offset","repo":"Tony-Y/pytorch_warmup","repo_kind":"listed","path":"pytorch_warmup/radam.py","file_url":"https://github.com/Tony-Y/pytorch_warmup/blob/HEAD/pytorch_warmup/radam.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4f02faccfbcfd169"}},{"code_sha256_prefix":"71f6aa75fb654ab1","entry":"get_warmup_params","repo":"Tony-Y/pytorch_warmup","repo_kind":"listed","path":"pytorch_warmup/base.py","file_url":"https://github.com/Tony-Y/pytorch_warmup/blob/HEAD/pytorch_warmup/base.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"71f6aa75fb654ab1"}},{"code_sha256_prefix":"920da29369351230","entry":"rho_fn","repo":"Tony-Y/pytorch_warmup","repo_kind":"listed","path":"pytorch_warmup/radam.py","file_url":"https://github.com/Tony-Y/pytorch_warmup/blob/HEAD/pytorch_warmup/radam.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"920da29369351230"}},{"code_sha256_prefix":"c6edb85f4f86bf17","entry":"rho_inf_fn","repo":"Tony-Y/pytorch_warmup","repo_kind":"listed","path":"pytorch_warmup/radam.py","file_url":"https://github.com/Tony-Y/pytorch_warmup/blob/HEAD/pytorch_warmup/radam.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c6edb85f4f86bf17"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}