{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/tree-of-thoughts-deliberate-problem-solving-1","title":"Tree of Thoughts: Deliberate Problem Solving with Large Language Models","arxiv_id":"2305.10601","date":"2023-05-17","proceeding":"NeurIPS 2023 11","authors":["Shunyu Yao","Dian Yu","Jeffrey Zhao","Izhak Shafran","Thomas L. Griffiths","Yuan Cao","Karthik Narasimhan"],"abstract":"Language models are increasingly being deployed for general problem solving across a wide range of tasks, but are still confined to token-level, left-to-right decision-making processes during inference. This means they can fall short in tasks that require exploration, strategic lookahead, or where initial decisions play a pivotal role. To surmount these challenges, we introduce a new framework for language model inference, Tree of Thoughts (ToT), which generalizes over the popular Chain of Thought approach to prompting language models, and enables exploration over coherent units of text (thoughts) that serve as intermediate steps toward problem solving. ToT allows LMs to perform deliberate decision making by considering multiple different reasoning paths and self-evaluating choices to decide the next course of action, as well as looking ahead or backtracking when necessary to make global choices. Our experiments show that ToT significantly enhances language models' problem-solving abilities on three novel tasks requiring non-trivial planning or search: Game of 24, Creative Writing, and Mini Crosswords. For instance, in Game of 24, while GPT-4 with chain-of-thought prompting only solved 4% of tasks, our method achieved a success rate of 74%. Code repo with all prompts: https://github.com/princeton-nlp/tree-of-thought-llm.","url_abs":"https://arxiv.org/abs/2305.10601v2","url_pdf":"https://arxiv.org/pdf/2305.10601v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"tree-of-thoughts-deliberate-problem-solving-1","repo_url":"https://github.com/princeton-nlp/tree-of-thought-llm","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"none","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"tree-of-thoughts-deliberate-problem-solving-1","repo_url":"https://github.com/ysymyth/tree-of-thought-llm","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"none","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"tree-of-thoughts-deliberate-problem-solving-1","repo_url":"https://github.com/appl-team/appl","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"tree-of-thoughts-deliberate-problem-solving-1","repo_url":"https://github.com/huiwy/reflection-on-trees","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":null},{"paper_slug":"tree-of-thoughts-deliberate-problem-solving-1","repo_url":"https://github.com/katamb/tree-of-thought-llm-ca","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"tree-of-thoughts-deliberate-problem-solving-1","repo_url":"https://github.com/codelion/optillm","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"arithmetic-reasoning","task_name":"Arithmetic Reasoning"},{"task_slug":"decision-making","task_name":"Decision Making"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"question-answering","task_name":"Question Answering"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt-4","method_name":"GPT-4"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/arithmetic-reasoning-on-game-of-24","task":"Arithmetic Reasoning","dataset":"Game of 24","model":"Tree of Thoughts (b=5)","rank_in_archive_order":1,"of":1,"metrics":{"Success":"0.74"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-truthfulqa","task":"Question Answering","dataset":"TruthfulQA","model":"ToT","rank_in_archive_order":30,"of":33,"metrics":{"EM":"66.6"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-webquestions","task":"Question Answering","dataset":"WebQuestions","model":"ToT","rank_in_archive_order":27,"of":37,"metrics":{"EM":"26.3"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2305.10601","atlas_url":"https://app.syntology.ai/?focus=2305.10601","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.10601"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/princeton-nlp/tree-of-thought-llm","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ysymyth/tree-of-thought-llm","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/appl-team/appl","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/katamb/tree-of-thought-llm-ca","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/codelion/optillm","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/huiwy/reflection-on-trees","reach":null}],"summary":{"ran":9,"ran_draft_wrong":1,"unverified":14},"by_repo_kind":{"official":{"samples":2,"ran":0,"repositories":1},"listed":{"samples":21,"ran":9,"repositories":4},"community":{"samples":1,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"4a6a2716cd1ecfd3","entry":"DialogueState","repo":"codelion/optillm","repo_kind":"listed","path":"optillm/mcts.py","file_url":"https://github.com/codelion/optillm/blob/HEAD/optillm/mcts.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4a6a2716cd1ecfd3"}},{"code_sha256_prefix":"c919c70de90b021d","entry":"MCTS","repo":"codelion/optillm","repo_kind":"listed","path":"optillm/mcts.py","file_url":"https://github.com/codelion/optillm/blob/HEAD/optillm/mcts.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c919c70de90b021d"}},{"code_sha256_prefix":"6445e3fedf96a2dd","entry":"MCTSAggregation","repo":"huiwy/reflection-on-trees","repo_kind":"listed","path":"reasoners/algorithm/mcts.py","file_url":"https://github.com/huiwy/reflection-on-trees/blob/HEAD/reasoners/algorithm/mcts.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6445e3fedf96a2dd"}},{"code_sha256_prefix":"778fb657e40c2423","entry":"MCTSNode","repo":"codelion/optillm","repo_kind":"listed","path":"optillm/mcts.py","file_url":"https://github.com/codelion/optillm/blob/HEAD/optillm/mcts.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"778fb657e40c2423"}},{"code_sha256_prefix":"e508098731faf687","entry":"MCTSNode","repo":"huiwy/reflection-on-trees","repo_kind":"listed","path":"reasoners/algorithm/mcts.py","file_url":"https://github.com/huiwy/reflection-on-trees/blob/HEAD/reasoners/algorithm/mcts.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e508098731faf687"}},{"code_sha256_prefix":"583801f0a2465d0c","entry":"MCTSResult","repo":"huiwy/reflection-on-trees","repo_kind":"listed","path":"reasoners/algorithm/mcts.py","file_url":"https://github.com/huiwy/reflection-on-trees/blob/HEAD/reasoners/algorithm/mcts.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"583801f0a2465d0c"}},{"code_sha256_prefix":"aee70aa81e74fa9a","entry":"clean_result","repo":"katamb/tree-of-thought-llm-ca","repo_kind":"listed","path":"src/tot/tasks/codeanalysis.py","file_url":"https://github.com/katamb/tree-of-thought-llm-ca/blob/HEAD/src/tot/tasks/codeanalysis.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"aee70aa81e74fa9a"}},{"code_sha256_prefix":"1b66502a5e6457a6","entry":"get_cwes","repo":"katamb/tree-of-thought-llm-ca","repo_kind":"listed","path":"src/tot/tasks/codeanalysis.py","file_url":"https://github.com/katamb/tree-of-thought-llm-ca/blob/HEAD/src/tot/tasks/codeanalysis.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1b66502a5e6457a6"}},{"code_sha256_prefix":"605644c3d7369622","entry":"gpt_usage","repo":"katamb/tree-of-thought-llm-ca","repo_kind":"listed","path":"src/tot/models.py","file_url":"https://github.com/katamb/tree-of-thought-llm-ca/blob/HEAD/src/tot/models.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"605644c3d7369622"}},{"code_sha256_prefix":"f55d3d712d45739b","entry":"string_to_dict","repo":"kyegomez/tree-of-thoughts","repo_kind":"community","path":"tree_of_thoughts/agent.py","file_url":"https://github.com/kyegomez/tree-of-thoughts/blob/HEAD/tree_of_thoughts/agent.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f55d3d712d45739b"}},{"code_sha256_prefix":"455bf868afcd3f83","entry":"AlgorithmOutput","repo":"huiwy/reflection-on-trees","repo_kind":"listed","path":"reasoners/algorithm/mcts.py","file_url":"https://github.com/huiwy/reflection-on-trees/blob/HEAD/reasoners/algorithm/mcts.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"455bf868afcd3f83"}},{"code_sha256_prefix":"a60a888f98f637a1","entry":"MCTS","repo":"huiwy/reflection-on-trees","repo_kind":"listed","path":"reasoners/algorithm/mcts.py","file_url":"https://github.com/huiwy/reflection-on-trees/blob/HEAD/reasoners/algorithm/mcts.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a60a888f98f637a1"}},{"code_sha256_prefix":"11e46ccab8c4b6b1","entry":"SearchAlgorithm","repo":"huiwy/reflection-on-trees","repo_kind":"listed","path":"reasoners/algorithm/mcts.py","file_url":"https://github.com/huiwy/reflection-on-trees/blob/HEAD/reasoners/algorithm/mcts.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"11e46ccab8c4b6b1"}},{"code_sha256_prefix":"4b65b29a47508556","entry":"SearchConfig","repo":"huiwy/reflection-on-trees","repo_kind":"listed","path":"reasoners/algorithm/mcts.py","file_url":"https://github.com/huiwy/reflection-on-trees/blob/HEAD/reasoners/algorithm/mcts.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4b65b29a47508556"}},{"code_sha256_prefix":"a843c8ac332d4291","entry":"WorldModel","repo":"huiwy/reflection-on-trees","repo_kind":"listed","path":"reasoners/algorithm/mcts.py","file_url":"https://github.com/huiwy/reflection-on-trees/blob/HEAD/reasoners/algorithm/mcts.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a843c8ac332d4291"}},{"code_sha256_prefix":"7eb3fea92922729c","entry":"appl_dedent","repo":"appl-team/appl","repo_kind":"listed","path":"src/appl/core/compile.py","file_url":"https://github.com/appl-team/appl/blob/HEAD/src/appl/core/compile.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7eb3fea92922729c"}},{"code_sha256_prefix":"9b3aebc249c853c6","entry":"dict_to_pydantic","repo":"appl-team/appl","repo_kind":"listed","path":"src/appl/caching/utils.py","file_url":"https://github.com/appl-team/appl/blob/HEAD/src/appl/caching/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9b3aebc249c853c6"}},{"code_sha256_prefix":"64959a06bee3598c","entry":"encode_to_uuid_v5","repo":"appl-team/appl","repo_kind":"listed","path":"src/appl/caching/utils.py","file_url":"https://github.com/appl-team/appl/blob/HEAD/src/appl/caching/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"64959a06bee3598c"}},{"code_sha256_prefix":"c4e580b088833502","entry":"find_dotenv","repo":"appl-team/appl","repo_kind":"listed","path":"src/appl/utils.py","file_url":"https://github.com/appl-team/appl/blob/HEAD/src/appl/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c4e580b088833502"}},{"code_sha256_prefix":"06226fb0ae82092a","entry":"find_files","repo":"appl-team/appl","repo_kind":"listed","path":"src/appl/utils.py","file_url":"https://github.com/appl-team/appl/blob/HEAD/src/appl/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"06226fb0ae82092a"}},{"code_sha256_prefix":"e453ebba492daf39","entry":"get_current_numbers","repo":"princeton-nlp/tree-of-thought-llm","repo_kind":"official","path":"src/tot/tasks/game24.py","file_url":"https://github.com/princeton-nlp/tree-of-thought-llm/blob/HEAD/src/tot/tasks/game24.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e453ebba492daf39"}},{"code_sha256_prefix":"70d9d6c9c2ec99d8","entry":"get_folder","repo":"appl-team/appl","repo_kind":"listed","path":"src/appl/utils.py","file_url":"https://github.com/appl-team/appl/blob/HEAD/src/appl/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"70d9d6c9c2ec99d8"}},{"code_sha256_prefix":"1506be9f936c7be7","entry":"gpt_usage","repo":"princeton-nlp/tree-of-thought-llm","repo_kind":"official","path":"src/tot/models.py","file_url":"https://github.com/princeton-nlp/tree-of-thought-llm/blob/HEAD/src/tot/models.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1506be9f936c7be7"}},{"code_sha256_prefix":"71622f179ee00cfd","entry":"pydantic_to_dict","repo":"appl-team/appl","repo_kind":"listed","path":"src/appl/caching/utils.py","file_url":"https://github.com/appl-team/appl/blob/HEAD/src/appl/caching/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"71622f179ee00cfd"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}