{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/revisiting-the-train-loss-an-efficient","title":"Speedy Performance Estimation for Neural Architecture Search","arxiv_id":"2006.04492","date":"2020-06-08","proceeding":"NeurIPS 2021 12","authors":["Binxin Ru","Clare Lyle","Lisa Schut","Miroslav Fil","Mark van der Wilk","Yarin Gal"],"abstract":"Reliable yet efficient evaluation of generalisation performance of a proposed architecture is crucial to the success of neural architecture search (NAS). Traditional approaches face a variety of limitations: training each architecture to completion is prohibitively expensive, early stopped validation accuracy may correlate poorly with fully trained performance, and model-based estimators require large training sets. We instead propose to estimate the final test performance based on a simple measure of training speed. Our estimator is theoretically motivated by the connection between generalisation and training speed, and is also inspired by the reformulation of a PAC-Bayes bound under the Bayesian setting. Our model-free estimator is simple, efficient, and cheap to implement, and does not require hyperparameter-tuning or surrogate training before deployment. We demonstrate on various NAS search spaces that our estimator consistently outperforms other alternatives in achieving better correlation with the true test performance rankings. We further show that our estimator can be easily incorporated into both query-based and one-shot NAS methods to improve the speed or quality of the search.","url_abs":"https://arxiv.org/abs/2006.04492v2","url_pdf":"https://arxiv.org/pdf/2006.04492v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"revisiting-the-train-loss-an-efficient","repo_url":"https://github.com/rubinxin/TSE","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":null},{"paper_slug":"revisiting-the-train-loss-an-efficient","repo_url":"https://github.com/rubinxin/SoTL","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}}],"tasks":[{"task_slug":"model-selection","task_name":"Model Selection"},{"task_slug":"architecture-search","task_name":"Neural Architecture Search"}],"methods":[{"method_slug":"early-stopping","method_name":"Early Stopping"},{"method_slug":"lstm","method_name":"LSTM"},{"method_slug":"speed","method_name":"SPEED"},{"method_slug":"sigmoid-activation","method_name":"Sigmoid Activation"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"tanh-activation","method_name":"Tanh Activation"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2006.04492","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2006.04492"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/rubinxin/TSE","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/rubinxin/SoTL","reach":{"status":"ok"}}],"summary":{"ran_honours":1,"unverified":1},"by_repo_kind":{"official":{"samples":1,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":1,"samples":[{"code_sha256_prefix":"17e87ce4fcceba76","entry":"train_and_eval","repo":"rubinxin/TSE","repo_kind":"official","path":"query_based_nas/run_regularized_evolution.py","file_url":"https://github.com/rubinxin/TSE/blob/HEAD/query_based_nas/run_regularized_evolution.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"17e87ce4fcceba76"}},{"code_sha256_prefix":"cb6cc47167b2300d","entry":"distill","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"cb6cc47167b2300d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}