{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/an-empirical-study-on-hyperparameter","title":"An Empirical Study on Hyperparameter Optimization for Fine-Tuning Pre-trained Language Models","arxiv_id":"2106.09204","date":"2021-06-17","proceeding":"ACL 2021 5","authors":["Xueqing Liu","Chi Wang"],"abstract":"The performance of fine-tuning pre-trained language models largely depends on the hyperparameter configuration. In this paper, we investigate the performance of modern hyperparameter optimization methods (HPO) on fine-tuning pre-trained language models. First, we study and report three HPO algorithms' performances on fine-tuning two state-of-the-art language models on the GLUE dataset. We find that using the same time budget, HPO often fails to outperform grid search due to two reasons: insufficient time budget and overfitting. We propose two general strategies and an experimental procedure to systematically troubleshoot HPO's failure cases. By applying the procedure, we observe that HPO can succeed with more appropriate settings in the search space and time budget; however, in certain cases overfitting remains. Finally, we make suggestions for future work. Our implementation can be found in https://github.com/microsoft/FLAML/tree/main/flaml/nlp/.","url_abs":"https://arxiv.org/abs/2106.09204v1","url_pdf":"https://arxiv.org/pdf/2106.09204v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"an-empirical-study-on-hyperparameter","repo_url":"https://github.com/microsoft/FLAML","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"none","reach":null}],"tasks":[{"task_slug":"hyperparameter-optimization","task_name":"Hyperparameter Optimization"}],"methods":[{"method_slug":"hpo","method_name":"HPO"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2106.09204","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2106.09204"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/microsoft/FLAML","reach":null},{"provenance":"deterministic:regex_extraction","url":"https://github.com/EvanLi/Github-Ranking","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_honours":1,"unverified":3},"by_repo_kind":{"found_in_text":{"samples":2,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":2,"samples":[{"code_sha256_prefix":"f29ed149e7409855","entry":"size","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"f29ed149e7409855"}},{"code_sha256_prefix":"9d114f6b99ff055b","entry":"get_api_repos","repo":"EvanLi/Github-Ranking","repo_kind":"found_in_text","path":"source/common.py","file_url":"https://github.com/EvanLi/Github-Ranking/blob/HEAD/source/common.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9d114f6b99ff055b"}},{"code_sha256_prefix":"ff30a5228ff375b8","entry":"get_graphql_data","repo":"EvanLi/Github-Ranking","repo_kind":"found_in_text","path":"source/common.py","file_url":"https://github.com/EvanLi/Github-Ranking/blob/HEAD/source/common.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ff30a5228ff375b8"}},{"code_sha256_prefix":"dc6c34254fe5f216","entry":"report","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"dc6c34254fe5f216"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}