{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/scaling-laws-and-compute-optimal-training","title":"Scaling Laws and Compute-Optimal Training Beyond Fixed Training Durations","arxiv_id":"2405.18392","date":"2024-05-28","proceeding":null,"authors":["Alexander Hägele","Elie Bakouch","Atli Kosson","Loubna Ben allal","Leandro von Werra","Martin Jaggi"],"abstract":"Scale has become a main ingredient in obtaining strong machine learning models. As a result, understanding a model's scaling properties is key to effectively designing both the right training setup as well as future generations of architectures. In this work, we argue that scale and training research has been needlessly complex due to reliance on the cosine schedule, which prevents training across different lengths for the same model size. We investigate the training behavior of a direct alternative -- constant learning rate and cooldowns -- and find that it scales predictably and reliably similar to cosine. Additionally, we show that stochastic weight averaging yields improved performance along the training trajectory, without additional training costs, across different scales. Importantly, with these findings we demonstrate that scaling experiments can be performed with significantly reduced compute and GPU hours by utilizing fewer but reusable training runs. Our code is available at \\url{https://github.com/epfml/schedules-and-scaling/}.","url_abs":"https://arxiv.org/abs/2405.18392v3","url_pdf":"https://arxiv.org/pdf/2405.18392v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"scaling-laws-and-compute-optimal-training","repo_url":"https://github.com/epfml/schedules-and-scaling","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"scaling-laws-and-compute-optimal-training","repo_url":"https://github.com/fabian-sp/lr-scheduling","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":null},{"paper_slug":"scaling-laws-and-compute-optimal-training","repo_url":"https://github.com/frotaur/icmlbackperp","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"scaling-laws-and-compute-optimal-training","repo_url":"https://github.com/frotaur/llm-arrows-of-time","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":null,"task_name":"GPU"}],"methods":[{"method_slug":"stochastic-weight-averaging","method_name":"Stochastic Weight Averaging"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2405.18392","atlas_url":"https://app.syntology.ai/?focus=2405.18392","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.18392"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/epfml/schedules-and-scaling","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/frotaur/llm-arrows-of-time","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/fabian-sp/lr-scheduling","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/frotaur/icmlbackperp","reach":{"status":"ok"}}],"summary":{"ran_draft_wrong":1,"ran":6,"unverified":6},"by_repo_kind":{"official":{"samples":8,"ran":3,"repositories":1},"listed":{"samples":5,"ran":4,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":4,"samples":[{"code_sha256_prefix":"c2122ff2161f0942","entry":"calculate_slope","repo":"fabian-sp/lr-scheduling","repo_kind":"listed","path":"reanalysis/analysis_horizon_transfer.py","file_url":"https://github.com/fabian-sp/lr-scheduling/blob/HEAD/reanalysis/analysis_horizon_transfer.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c2122ff2161f0942"}},{"code_sha256_prefix":"1bd86a911c3f78a3","entry":"move_to_cpu","repo":"epfml/schedules-and-scaling","repo_kind":"official","path":"src/logger/logger.py","file_url":"https://github.com/epfml/schedules-and-scaling/blob/HEAD/src/logger/logger.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1bd86a911c3f78a3"}},{"code_sha256_prefix":"db9d402a23bb81a3","entry":"multiget","repo":"frotaur/icmlbackperp","repo_kind":"listed","path":"modules/tokenizer.py","file_url":"https://github.com/frotaur/icmlbackperp/blob/HEAD/modules/tokenizer.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"db9d402a23bb81a3"}},{"code_sha256_prefix":"a1d6f89d43fc42e1","entry":"precompute_freqs_cis","repo":"epfml/schedules-and-scaling","repo_kind":"official","path":"src/models/llama.py","file_url":"https://github.com/epfml/schedules-and-scaling/blob/HEAD/src/models/llama.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a1d6f89d43fc42e1"}},{"code_sha256_prefix":"e21c63d05fe20e51","entry":"put_and_return","repo":"frotaur/icmlbackperp","repo_kind":"listed","path":"modules/tokenizer.py","file_url":"https://github.com/frotaur/icmlbackperp/blob/HEAD/modules/tokenizer.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e21c63d05fe20e51"}},{"code_sha256_prefix":"b5644fc0a380fa78","entry":"remove_none_vals","repo":"frotaur/icmlbackperp","repo_kind":"listed","path":"modules/tokenizer.py","file_url":"https://github.com/frotaur/icmlbackperp/blob/HEAD/modules/tokenizer.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b5644fc0a380fa78"}},{"code_sha256_prefix":"198aea734522a443","entry":"self_preserving_overwrite","repo":"epfml/schedules-and-scaling","repo_kind":"official","path":"src/logger/logger.py","file_url":"https://github.com/epfml/schedules-and-scaling/blob/HEAD/src/logger/logger.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"198aea734522a443"}},{"code_sha256_prefix":"9891174b43bd9c9a","entry":"apply_rotary_emb","repo":"epfml/schedules-and-scaling","repo_kind":"official","path":"src/models/llama.py","file_url":"https://github.com/epfml/schedules-and-scaling/blob/HEAD/src/models/llama.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9891174b43bd9c9a"}},{"code_sha256_prefix":"91d9a7a7d597054b","entry":"cos_inf_schedule","repo":"epfml/schedules-and-scaling","repo_kind":"official","path":"src/optim/utils.py","file_url":"https://github.com/epfml/schedules-and-scaling/blob/HEAD/src/optim/utils.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"91d9a7a7d597054b"}},{"code_sha256_prefix":"f9dc20d4ad58a5f9","entry":"get_batch","repo":"epfml/schedules-and-scaling","repo_kind":"official","path":"src/optim/utils.py","file_url":"https://github.com/epfml/schedules-and-scaling/blob/HEAD/src/optim/utils.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f9dc20d4ad58a5f9"}},{"code_sha256_prefix":"843859340885ab12","entry":"get_model","repo":"epfml/schedules-and-scaling","repo_kind":"official","path":"src/models/utils.py","file_url":"https://github.com/epfml/schedules-and-scaling/blob/HEAD/src/models/utils.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"843859340885ab12"}},{"code_sha256_prefix":"98497683abdc8f17","entry":"load_model","repo":"frotaur/icmlbackperp","repo_kind":"listed","path":"modules/models/load_model.py","file_url":"https://github.com/frotaur/icmlbackperp/blob/HEAD/modules/models/load_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"98497683abdc8f17"}},{"code_sha256_prefix":"e8ac769090ecbe45","entry":"wsd_schedule","repo":"epfml/schedules-and-scaling","repo_kind":"official","path":"src/optim/utils.py","file_url":"https://github.com/epfml/schedules-and-scaling/blob/HEAD/src/optim/utils.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e8ac769090ecbe45"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}