{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/large-language-models-as-optimizers","title":"Large Language Models as Optimizers","arxiv_id":"2309.03409","date":"2023-09-07","proceeding":null,"authors":["Chengrun Yang","Xuezhi Wang","Yifeng Lu","Hanxiao Liu","Quoc V. Le","Denny Zhou","Xinyun Chen"],"abstract":"Optimization is ubiquitous. While derivative-based algorithms have been powerful tools for various problems, the absence of gradient imposes challenges on many real-world applications. In this work, we propose Optimization by PROmpting (OPRO), a simple and effective approach to leverage large language models (LLMs) as optimizers, where the optimization task is described in natural language. In each optimization step, the LLM generates new solutions from the prompt that contains previously generated solutions with their values, then the new solutions are evaluated and added to the prompt for the next optimization step. We first showcase OPRO on linear regression and traveling salesman problems, then move on to our main application in prompt optimization, where the goal is to find instructions that maximize the task accuracy. With a variety of LLMs, we demonstrate that the best prompts optimized by OPRO outperform human-designed prompts by up to 8% on GSM8K, and by up to 50% on Big-Bench Hard tasks. Code at https://github.com/google-deepmind/opro.","url_abs":"https://arxiv.org/abs/2309.03409v3","url_pdf":"https://arxiv.org/pdf/2309.03409v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"large-language-models-as-optimizers","repo_url":"https://github.com/google-deepmind/opro","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"large-language-models-as-optimizers","repo_url":"https://github.com/finitearth/promptolution","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"large-language-models-as-optimizers","repo_url":"https://github.com/thunderous77/glape","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"large-language-models-as-optimizers","repo_url":"https://github.com/yaolu/random-prompt","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"gsm8k","task_name":"GSM8K"}],"methods":[{"method_slug":"linear-regression","method_name":"Linear Regression"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2309.03409","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2309.03409"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/google-deepmind/opro","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/thunderous77/glape","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/finitearth/promptolution","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yaolu/random-prompt","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":3,"ran":8,"ran_fixture":1,"unverified":2},"by_repo_kind":{"official":{"samples":9,"ran":7,"repositories":1},"listed":{"samples":5,"ran":5,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":2,"samples":[{"code_sha256_prefix":"a4cf9d0d1ece601e","entry":"read_jsonl","repo":"google-deepmind/opro","repo_kind":"official","path":"opro/evaluation/eval_utils.py","file_url":"https://github.com/google-deepmind/opro/blob/HEAD/opro/evaluation/eval_utils.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":2,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a4cf9d0d1ece601e"}},{"code_sha256_prefix":"19d3b6894577dc43","entry":"balanced_sample","repo":"yaolu/random-prompt","repo_kind":"listed","path":"utils.py","file_url":"https://github.com/yaolu/random-prompt/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"19d3b6894577dc43"}},{"code_sha256_prefix":"ab2577b54a5de0f2","entry":"compute_loss","repo":"thunderous77/glape","repo_kind":"listed","path":"utils.py","file_url":"https://github.com/thunderous77/glape/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ab2577b54a5de0f2"}},{"code_sha256_prefix":"9e6b4f1e39187226","entry":"extract_string_in_square_brackets","repo":"google-deepmind/opro","repo_kind":"official","path":"opro/optimization/opt_utils.py","file_url":"https://github.com/google-deepmind/opro/blob/HEAD/opro/optimization/opt_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9e6b4f1e39187226"}},{"code_sha256_prefix":"d3c113af9902217e","entry":"gen_ins_and_score_pairs_substr","repo":"google-deepmind/opro","repo_kind":"official","path":"opro/optimization/opt_utils.py","file_url":"https://github.com/google-deepmind/opro/blob/HEAD/opro/optimization/opt_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d3c113af9902217e"}},{"code_sha256_prefix":"355c3b041e1b77a5","entry":"get_context_limit","repo":"yaolu/random-prompt","repo_kind":"listed","path":"utils.py","file_url":"https://github.com/yaolu/random-prompt/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"355c3b041e1b77a5"}},{"code_sha256_prefix":"22e0818b777cfffa","entry":"instruction_to_filename","repo":"google-deepmind/opro","repo_kind":"official","path":"opro/evaluation/eval_utils.py","file_url":"https://github.com/google-deepmind/opro/blob/HEAD/opro/evaluation/eval_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"22e0818b777cfffa"}},{"code_sha256_prefix":"a3f7ac8674bc5e72","entry":"optimize","repo":"thunderous77/glape","repo_kind":"listed","path":"utils.py","file_url":"https://github.com/thunderous77/glape/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a3f7ac8674bc5e72"}},{"code_sha256_prefix":"2bd751a94e813601","entry":"parse_tag_content","repo":"google-deepmind/opro","repo_kind":"official","path":"opro/optimization/opt_utils.py","file_url":"https://github.com/google-deepmind/opro/blob/HEAD/opro/optimization/opt_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2bd751a94e813601"}},{"code_sha256_prefix":"77a2c584ec1b5055","entry":"profile_time","repo":"yaolu/random-prompt","repo_kind":"listed","path":"utils.py","file_url":"https://github.com/yaolu/random-prompt/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"77a2c584ec1b5055"}},{"code_sha256_prefix":"dc0fb1ce4824c2c3","entry":"remove_punctuation_from_string","repo":"google-deepmind/opro","repo_kind":"official","path":"opro/evaluation/eval_utils.py","file_url":"https://github.com/google-deepmind/opro/blob/HEAD/opro/evaluation/eval_utils.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"dc0fb1ce4824c2c3"}},{"code_sha256_prefix":"f980d622248dbd94","entry":"remove_punctuation_from_string","repo":"google-deepmind/opro","repo_kind":"official","path":"opro/evaluation/metrics.py","file_url":"https://github.com/google-deepmind/opro/blob/HEAD/opro/evaluation/metrics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f980d622248dbd94"}},{"code_sha256_prefix":"310edb478a4009a1","entry":"call_openai_server_func","repo":"google-deepmind/opro","repo_kind":"official","path":"opro/prompt_utils.py","file_url":"https://github.com/google-deepmind/opro/blob/HEAD/opro/prompt_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"310edb478a4009a1"}},{"code_sha256_prefix":"daa969a73f4f1e26","entry":"call_openai_server_single_prompt","repo":"google-deepmind/opro","repo_kind":"official","path":"opro/prompt_utils.py","file_url":"https://github.com/google-deepmind/opro/blob/HEAD/opro/prompt_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"daa969a73f4f1e26"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}