{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/no-train-no-gain-revisiting-efficient","title":"No Train No Gain: Revisiting Efficient Training Algorithms For Transformer-based Language Models","arxiv_id":"2307.06440","date":"2023-07-12","proceeding":"NeurIPS 2023 11","authors":["Jean Kaddour","Oscar Key","Piotr Nawrot","Pasquale Minervini","Matt J. Kusner"],"abstract":"The computation necessary for training Transformer-based language models has skyrocketed in recent years. This trend has motivated research on efficient training algorithms designed to improve training, validation, and downstream performance faster than standard training. In this work, we revisit three categories of such algorithms: dynamic architectures (layer stacking, layer dropping), batch selection (selective backprop, RHO loss), and efficient optimizers (Lion, Sophia). When pre-training BERT and T5 with a fixed computation budget using such methods, we find that their training, validation, and downstream gains vanish compared to a baseline with a fully-decayed learning rate. We define an evaluation protocol that enables computation to be done on arbitrary machines by mapping all computation time to a reference machine which we call reference system time. We discuss the limitations of our proposed protocol and release our code to encourage rigorous research in efficient training procedures: https://github.com/JeanKaddour/NoTrainNoGain.","url_abs":"https://arxiv.org/abs/2307.06440v4","url_pdf":"https://arxiv.org/pdf/2307.06440v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"no-train-no-gain-revisiting-efficient","repo_url":"https://github.com/jeankaddour/notrainnogain","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[],"methods":[{"method_slug":"adafactor","method_name":"Adafactor"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"glu","method_name":"Gated Linear Unit"},{"method_slug":"inverse-square-root-schedule","method_name":"Inverse Square Root Schedule"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"sentencepiece","method_name":"SentencePiece"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"t5","method_name":"T5"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2307.06440","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2307.06440"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/JeanKaddour/NoTrainNoGain","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jeankaddour/notrainnogain","reach":{"status":"ok"}}],"summary":{"ran_violates":3,"ran":7,"unverified":3},"by_repo_kind":{"official":{"samples":12,"ran":9,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":13,"samples":[{"code_sha256_prefix":"a7137af5385dbe09","entry":"check_deadline","repo":"JeanKaddour/NoTrainNoGain","repo_kind":"official","path":"bert/pretrain_bert_rho_loss.py","file_url":"https://github.com/JeanKaddour/NoTrainNoGain/blob/HEAD/bert/pretrain_bert_rho_loss.py","link_basis":"plan_row","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a7137af5385dbe09"}},{"code_sha256_prefix":"5bf52ae7f2f2317a","entry":"check_early_termination","repo":"JeanKaddour/NoTrainNoGain","repo_kind":"official","path":"bert/pretrain_bert_rho_loss.py","file_url":"https://github.com/JeanKaddour/NoTrainNoGain/blob/HEAD/bert/pretrain_bert_rho_loss.py","link_basis":"plan_row","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5bf52ae7f2f2317a"}},{"code_sha256_prefix":"aa5486a3650902d8","entry":"exists","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"aa5486a3650902d8"}},{"code_sha256_prefix":"2840d349f4c8b0c1","entry":"get_example_ids_from_batch","repo":"JeanKaddour/NoTrainNoGain","repo_kind":"official","path":"bert/efficient_training/extract_il_losses.py","file_url":"https://github.com/JeanKaddour/NoTrainNoGain/blob/HEAD/bert/efficient_training/extract_il_losses.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2840d349f4c8b0c1"}},{"code_sha256_prefix":"4681043cc92ad4c3","entry":"get_layer_fn","repo":"JeanKaddour/NoTrainNoGain","repo_kind":"official","path":"bert/cramming/architectures/fused_layers.py","file_url":"https://github.com/JeanKaddour/NoTrainNoGain/blob/HEAD/bert/cramming/architectures/fused_layers.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4681043cc92ad4c3"}},{"code_sha256_prefix":"98dec28a22e6c58a","entry":"layer_structure","repo":"JeanKaddour/NoTrainNoGain","repo_kind":"official","path":"bert/cramming/architectures/fused_layers.py","file_url":"https://github.com/JeanKaddour/NoTrainNoGain/blob/HEAD/bert/cramming/architectures/fused_layers.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"98dec28a22e6c58a"}},{"code_sha256_prefix":"f048db3c79eb5ee8","entry":"load_chunks","repo":"JeanKaddour/NoTrainNoGain","repo_kind":"official","path":"bert/pretrain_bert_rho_loss.py","file_url":"https://github.com/JeanKaddour/NoTrainNoGain/blob/HEAD/bert/pretrain_bert_rho_loss.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f048db3c79eb5ee8"}},{"code_sha256_prefix":"276a79af4663120e","entry":"sample_active_layers","repo":"JeanKaddour/NoTrainNoGain","repo_kind":"official","path":"bert/efficient_training/layer_drop.py","file_url":"https://github.com/JeanKaddour/NoTrainNoGain/blob/HEAD/bert/efficient_training/layer_drop.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"276a79af4663120e"}},{"code_sha256_prefix":"79efd868ef38d1ad","entry":"scaled_layer_training","repo":"JeanKaddour/NoTrainNoGain","repo_kind":"official","path":"bert/cramming/architectures/fused_layers.py","file_url":"https://github.com/JeanKaddour/NoTrainNoGain/blob/HEAD/bert/cramming/architectures/fused_layers.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"79efd868ef38d1ad"}},{"code_sha256_prefix":"d782f0fce6dc8a6b","entry":"within_budget","repo":"JeanKaddour/NoTrainNoGain","repo_kind":"official","path":"bert/pretrain_bert.py","file_url":"https://github.com/JeanKaddour/NoTrainNoGain/blob/HEAD/bert/pretrain_bert.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d782f0fce6dc8a6b"}},{"code_sha256_prefix":"a054d17634dd4dfd","entry":"find_pretrained_checkpoint","repo":"JeanKaddour/NoTrainNoGain","repo_kind":"official","path":"bert/cramming/utils.py","file_url":"https://github.com/JeanKaddour/NoTrainNoGain/blob/HEAD/bert/cramming/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a054d17634dd4dfd"}},{"code_sha256_prefix":"02ca444386b6a0b5","entry":"save_summary","repo":"JeanKaddour/NoTrainNoGain","repo_kind":"official","path":"bert/cramming/utils.py","file_url":"https://github.com/JeanKaddour/NoTrainNoGain/blob/HEAD/bert/cramming/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"02ca444386b6a0b5"}},{"code_sha256_prefix":"871963e556958789","entry":"system_startup","repo":"JeanKaddour/NoTrainNoGain","repo_kind":"official","path":"bert/cramming/utils.py","file_url":"https://github.com/JeanKaddour/NoTrainNoGain/blob/HEAD/bert/cramming/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"871963e556958789"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}