{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/verified-multi-step-synthesis-using-large","title":"VerMCTS: Synthesizing Multi-Step Programs using a Verifier, a Large Language Model, and Tree Search","arxiv_id":"2402.08147","date":"2024-02-13","proceeding":null,"authors":["David Brandfonbrener","Simon Henniger","Sibi Raja","Tarun Prasad","Chloe Loughridge","Federico Cassano","Sabrina Ruixin Hu","Jianang Yang","William E. Byrd","Robert Zinkov","Nada Amin"],"abstract":"Large Language Models (LLMs) can generate useful code, but often the code they generate cannot be trusted to be sound. In this paper, we present VerMCTS, an approach to begin to resolve this issue by generating verified programs in Dafny and Coq. VerMCTS uses a logical verifier in concert with an LLM to guide a modified Monte Carlo Tree Search (MCTS). This approach leverages the verifier to gain intermediate feedback inside the search algorithm by checking partial programs at each step to estimate an upper bound on the value function. To measure the performance of VerMCTS, we develop a new suite of multi-step verified programming problems in Dafny and Coq. In terms of pass@T, a new metric which computes the pass rate given a budget of T tokens sampled from the LLM, VerMCTS leads to more than a 30% absolute increase in average pass@5000 across the suite over repeated sampling from the base language model. Our code and benchmarks are available at https://github.com/namin/llm-verified-with-monte-carlo-tree-search .","url_abs":"https://arxiv.org/abs/2402.08147v2","url_pdf":"https://arxiv.org/pdf/2402.08147v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"verified-multi-step-synthesis-using-large","repo_url":"https://github.com/namin/llm-verified-with-monte-carlo-tree-search","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"large-language-model","task_name":"Large Language Model"}],"methods":[{"method_slug":"base","method_name":"BASE"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2402.08147","atlas_url":"https://app.syntology.ai/?focus=2402.08147","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.08147"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/namin/llm-verified-with-monte-carlo-tree-search","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":8},"by_repo_kind":{"official":{"samples":8,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"0bbdad54ec776d84","entry":"check_func","repo":"namin/llm-verified-with-monte-carlo-tree-search","repo_kind":"official","path":"clover_prompts.py","file_url":"https://github.com/namin/llm-verified-with-monte-carlo-tree-search/blob/HEAD/clover_prompts.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0bbdad54ec776d84"}},{"code_sha256_prefix":"dceb3bd46b3111d3","entry":"count_depth","repo":"namin/llm-verified-with-monte-carlo-tree-search","repo_kind":"official","path":"common.py","file_url":"https://github.com/namin/llm-verified-with-monte-carlo-tree-search/blob/HEAD/common.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dceb3bd46b3111d3"}},{"code_sha256_prefix":"f537b02e6b477d7d","entry":"create_score_predicate","repo":"namin/llm-verified-with-monte-carlo-tree-search","repo_kind":"official","path":"common_cache.py","file_url":"https://github.com/namin/llm-verified-with-monte-carlo-tree-search/blob/HEAD/common_cache.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f537b02e6b477d7d"}},{"code_sha256_prefix":"5928e0e0d10f0c1c","entry":"messages_to_prompt","repo":"namin/llm-verified-with-monte-carlo-tree-search","repo_kind":"official","path":"call_rag.py","file_url":"https://github.com/namin/llm-verified-with-monte-carlo-tree-search/blob/HEAD/call_rag.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5928e0e0d10f0c1c"}},{"code_sha256_prefix":"9608c6c38406c666","entry":"query_keywords","repo":"namin/llm-verified-with-monte-carlo-tree-search","repo_kind":"official","path":"call_rag.py","file_url":"https://github.com/namin/llm-verified-with-monte-carlo-tree-search/blob/HEAD/call_rag.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9608c6c38406c666"}},{"code_sha256_prefix":"888b8f44cb626ee0","entry":"score_first","repo":"namin/llm-verified-with-monte-carlo-tree-search","repo_kind":"official","path":"common_cache.py","file_url":"https://github.com/namin/llm-verified-with-monte-carlo-tree-search/blob/HEAD/common_cache.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"888b8f44cb626ee0"}},{"code_sha256_prefix":"68989aabf14b371b","entry":"select_diversely","repo":"namin/llm-verified-with-monte-carlo-tree-search","repo_kind":"official","path":"common_diversity.py","file_url":"https://github.com/namin/llm-verified-with-monte-carlo-tree-search/blob/HEAD/common_diversity.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"68989aabf14b371b"}},{"code_sha256_prefix":"123e42c32ccbf108","entry":"select_with_scores","repo":"namin/llm-verified-with-monte-carlo-tree-search","repo_kind":"official","path":"common_cache.py","file_url":"https://github.com/namin/llm-verified-with-monte-carlo-tree-search/blob/HEAD/common_cache.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"123e42c32ccbf108"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}