{"url":"/method/linear-warmup","slug":"linear-warmup","name":"Linear Warmup","full_name":"Linear Warmup","full_name_withheld":false,"description_markdown":"**Linear Warmup** is a learning rate schedule where we linearly increase the learning rate from a low rate to a constant rate thereafter. This reduces volatility in the early stages of training.\r\n\r\nImage Credit: [Chengwei Zhang](https://www.dlology.com/about-me/)","description_state":"present","introduced_year":null,"introduced_by":{"title":null,"paper":null,"first_author":null,"n_authors":0,"url_abs":null,"archive_paper_url":null},"source":{"url":null,"title":null,"url_on_a_paper_host":false},"code_snippet_url":null,"code_snippet_url_on_a_code_host":false,"categories":[{"area":"General","area_id":"general","collection":"Learning Rate Schedules","url":"/methods/category/learning-rate-schedules","pwc_aliases":[]}],"n_papers_tagged":38,"archive_num_papers":38,"papers_newest_first":[{"paper":"/paper/the-warmup-dilemma-how-learning-rate","title":"The Warmup Dilemma: How Learning Rate Strategies Impact Speech-to-Text Model Convergence","date":"2025-05-29","arxiv_id":"2505.23420","n_code_links":1,"syntology":null},{"paper":"/paper/sample-and-computationally-efficient-1","title":"Sample and Computationally Efficient Continuous-Time Reinforcement Learning with General Function Approximation","date":"2025-05-20","arxiv_id":"2505.14821","n_code_links":1,"syntology":null},{"paper":null,"title":"Tractable Representations for Convergent Approximation of Distributional HJB Equations","date":"2025-03-07","arxiv_id":"2503.05563","n_code_links":0,"syntology":null},{"paper":null,"title":"Integrating LLMs with ITS: Recent Advances, Potentials, Challenges, and Future Directions","date":"2025-01-08","arxiv_id":"2501.04437","n_code_links":0,"syntology":null},{"paper":null,"title":"A Combined Encoder and Transformer Approach for Coherent and High-Quality Text Generation","date":"2024-11-19","arxiv_id":"2411.12157","n_code_links":0,"syntology":null},{"paper":"/paper/causal-temporal-representation-learning-with","title":"Causal Temporal Representation Learning with Nonstationary Sparse Transition","date":"2024-09-05","arxiv_id":"2409.03142","n_code_links":1,"syntology":{"ran":7,"of":9,"unverified":2,"pointer_only":9}},{"paper":"/paper/machine-learning-models-for-daily-rainfall","title":"Machine learning models for daily rainfall forecasting in Northern Tropical Africa using tropical wave predictors","date":"2024-08-29","arxiv_id":"2408.16349","n_code_links":1,"syntology":null},{"paper":null,"title":"CTRL: Continuous-Time Representation Learning on Temporal Heterogeneous Information Network","date":"2024-05-11","arxiv_id":"2405.08013","n_code_links":0,"syntology":null},{"paper":"/paper/an-embarrassingly-simple-defense-against","title":"Towards Adversarial Robustness And Backdoor Mitigation in SSL","date":"2024-03-23","arxiv_id":"2403.15918","n_code_links":1,"syntology":null},{"paper":"/paper/two-trades-is-not-baffled-condense-graph-via","title":"Two Trades is not Baffled: Condensing Graph via Crafting Rational Gradient Matching","date":"2024-02-07","arxiv_id":"2402.04924","n_code_links":1,"syntology":{"ran":3,"of":3,"unverified":0,"pointer_only":0}},{"paper":"/paper/continual-pre-training-of-large-language","title":"Continual Pre-Training of Large Language Models: How to (re)warm your model?","date":"2023-08-08","arxiv_id":"2308.04014","n_code_links":2,"syntology":null},{"paper":null,"title":"CTRL: Connect Collaborative and Language Model for CTR Prediction","date":"2023-06-05","arxiv_id":"2306.02841","n_code_links":0,"syntology":null},{"paper":"/paper/swectrl-mini-a-data-transparent-transformer","title":"SweCTRL-Mini: a data-transparent Transformer-based large language model for controllable text generation in Swedish","date":"2023-04-27","arxiv_id":"2304.13994","n_code_links":1,"syntology":null},{"paper":"/paper/once-detected-never-lost-surpassing-human","title":"Once Detected, Never Lost: Surpassing Human Performance in Offline LiDAR based 3D Object Detection","date":"2023-04-24","arxiv_id":"2304.12315","n_code_links":2,"syntology":{"ran":3,"of":5,"unverified":2,"pointer_only":0}},{"paper":"/paper/elastic-weight-removal-for-faithful-and","title":"Elastic Weight Removal for Faithful and Abstractive Dialogue Generation","date":"2023-03-30","arxiv_id":"2303.17574","n_code_links":1,"syntology":null},{"paper":null,"title":"Mixing Backward- with Forward-Chaining for Metacognitive Skill Acquisition and Transfer","date":"2023-03-18","arxiv_id":"2303.12223","n_code_links":0,"syntology":null},{"paper":null,"title":"Alternative formulations for gilthead seabream diets: towards a more sustainable production","date":"2022-11-03","arxiv_id":"2211.02430","n_code_links":0,"syntology":null},{"paper":"/paper/controllable-factuality-in-document-grounded","title":"Controllable Factuality in Document-Grounded Dialog Systems Using a Noisy Channel Model","date":"2022-10-31","arxiv_id":"2210.17418","n_code_links":1,"syntology":null},{"paper":"/paper/unsupervised-learning-of-structured","title":"Unsupervised Learning of Structured Representations via Closed-Loop Transcription","date":"2022-10-30","arxiv_id":"2210.16782","n_code_links":1,"syntology":null},{"paper":"/paper/demystifying-self-supervised-trojan-attacks","title":"An Embarrassingly Simple Backdoor Attack on Self-supervised Learning","date":"2022-10-13","arxiv_id":"2210.07346","n_code_links":4,"syntology":{"ran":2,"of":2,"unverified":0,"pointer_only":2}},{"paper":"/paper/continual-learning-fast-and-slow","title":"Continual Learning, Fast and Slow","date":"2022-09-06","arxiv_id":"2209.02370","n_code_links":1,"syntology":null},{"paper":"/paper/ctrl-clustering-training-losses-for-label","title":"CTRL: Clustering Training Losses for Label Error Detection","date":"2022-08-17","arxiv_id":"2208.08464","n_code_links":1,"syntology":null},{"paper":"/paper/pursuit-of-a-discriminative-representation","title":"Pursuit of a Discriminative Representation for Multiple Subspaces via Sequential Games","date":"2022-06-18","arxiv_id":"2206.09120","n_code_links":1,"syntology":{"ran":0,"of":1,"unverified":1,"pointer_only":0}},{"paper":null,"title":"Efficient and Training-Free Control of Language Generation","date":"2022-05-12","arxiv_id":"2205.06036","n_code_links":0,"syntology":null},{"paper":null,"title":"Gradient Descent, Stochastic Optimization, and Other Tales","date":"2022-05-02","arxiv_id":"2205.00832","n_code_links":0,"syntology":null},{"paper":null,"title":"A Comparative Study of Transformers on Word Sense Disambiguation","date":"2021-11-30","arxiv_id":"2111.15417","n_code_links":0,"syntology":null},{"paper":"/paper/turingbench-a-benchmark-environment-for","title":"TURINGBENCH: A Benchmark Environment for Turing Test in the Age of Neural Text Generation","date":"2021-09-27","arxiv_id":"2109.13296","n_code_links":3,"syntology":{"ran":0,"of":7,"unverified":7,"pointer_only":0}},{"paper":"/paper/cross-trajectory-representation-learning-for","title":"Cross-Trajectory Representation Learning for Zero-Shot Generalization in RL","date":"2021-06-04","arxiv_id":"2106.02193","n_code_links":1,"syntology":{"ran":7,"of":11,"unverified":4,"pointer_only":11}},{"paper":"/paper/learning-to-relate-depth-and-semantics-for","title":"Learning to Relate Depth and Semantics for Unsupervised Domain Adaptation","date":"2021-05-17","arxiv_id":"2105.07830","n_code_links":1,"syntology":{"ran":3,"of":4,"unverified":1,"pointer_only":4}},{"paper":"/paper/ctlr-wic-tsv-target-sense-verification-using","title":"CTLR@WiC-TSV: Target Sense Verification using Marked Inputs andPre-trained Models","date":"2021-04-30","arxiv_id":null,"n_code_links":0,"syntology":null}],"papers_shown":30,"tasks":[{"task":"/task/language-modelling","name":"Language Modelling","papers":7},{"task":"/task/representation-learning","name":"Representation Learning","papers":6},{"task":"/task/text-generation","name":"Text Generation","papers":5},{"task":"/task/language-modeling","name":"Language Modeling","papers":4},{"task":"/task/object","name":"Object","papers":3},{"task":"/task/object-detection","name":"Object Detection","papers":3},{"task":"/task/self-supervised-learning","name":"Self-Supervised Learning","papers":3},{"task":"/task/adversarial-robustness","name":"Adversarial Robustness","papers":2},{"task":"/task/backdoor-attack","name":"Backdoor Attack","papers":2},{"task":null,"name":"GPU","papers":2},{"task":"/task/image-classification","name":"Image Classification","papers":2},{"task":"/task/reinforcement-learning-1","name":"Reinforcement Learning (RL)","papers":2},{"task":"/task/segmentation","name":"Segmentation","papers":2},{"task":"/task/semantic-segmentation","name":"Semantic Segmentation","papers":2},{"task":"/task/object-detection-1","name":"object-detection","papers":2},{"task":"/task/3d-object-detection","name":"3D Object Detection","papers":1},{"task":"/task/action-classification","name":"Action Classification","papers":1},{"task":"/task/action-recognition-in-videos","name":"Action Recognition","papers":1},{"task":"/task/attribute","name":"Attribute","papers":1},{"task":"/task/authorship-attribution","name":"Authorship Attribution","papers":1}],"tasks_shown":20,"n_tasks":79,"usage_by_year":[{"year":"2017","papers":1},{"year":"2018","papers":1},{"year":"2019","papers":4},{"year":"2020","papers":2},{"year":"2021","papers":5},{"year":"2022","papers":9},{"year":"2023","papers":6},{"year":"2024","papers":6},{"year":"2025","papers":4}],"row_source":"methods_table","archive":{"source":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","archive_url":"https://paperswithcode.com/method/linear-warmup"},"syntology_read_at":"2026-09-24T18:15:14+00:00"}