{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/2505-10981","title":"Rethinking the Role of Prompting Strategies in LLM Test-Time Scaling: A Perspective of Probability Theory","arxiv_id":"2505.10981","date":"2025-05-16","proceeding":null,"authors":["Yexiang Liu","Zekun Li","Zhi Fang","Nan Xu","Ran He","Tieniu Tan"],"abstract":"Recently, scaling test-time compute on Large Language Models (LLM) has garnered wide attention. However, there has been limited investigation of how various reasoning prompting strategies perform as scaling. In this paper, we focus on a standard and realistic scaling setting: majority voting. We systematically conduct experiments on 6 LLMs $\\times$ 8 prompting strategies $\\times$ 6 benchmarks. Experiment results consistently show that as the sampling time and computational overhead increase, complicated prompting strategies with superior initial performance gradually fall behind simple Chain-of-Thought. We analyze this phenomenon and provide theoretical proofs. Additionally, we propose a method according to probability theory to quickly and accurately predict the scaling performance and select the best strategy under large sampling times without extra resource-intensive inference in practice. It can serve as the test-time scaling law for majority voting. Furthermore, we introduce two ways derived from our theoretical analysis to significantly improve the scaling performance. We hope that our research can promote to re-examine the role of complicated prompting, unleash the potential of simple prompting strategies, and provide new insights for enhancing test-time scaling performance.","url_abs":"https://arxiv.org/abs/2505.10981v1","url_pdf":"https://arxiv.org/pdf/2505.10981v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"2505-10981","repo_url":"https://github.com/MraDonkey/rethinking_prompting","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":null}],"tasks":[],"methods":[{"method_slug":"focus","method_name":"Focus"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2505.10981","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.10981"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/MraDonkey/rethinking_prompting","reach":null}],"summary":{"ran_draft_wrong":3,"ran_honours":1,"unverified":2},"by_repo_kind":{"official":{"samples":6,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"271c0d63a2ec91c7","entry":"gpt_parallel_generate","repo":"MraDonkey/rethinking_prompting","repo_kind":"official","path":"model.py","file_url":"https://github.com/MraDonkey/rethinking_prompting/blob/HEAD/model.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"271c0d63a2ec91c7"}},{"code_sha256_prefix":"0b14c648516c38a7","entry":"last_boxed_only_string","repo":"MraDonkey/rethinking_prompting","repo_kind":"official","path":"model.py","file_url":"https://github.com/MraDonkey/rethinking_prompting/blob/HEAD/model.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0b14c648516c38a7"}},{"code_sha256_prefix":"8686c0a944293a75","entry":"parse_answer","repo":"MraDonkey/rethinking_prompting","repo_kind":"official","path":"model.py","file_url":"https://github.com/MraDonkey/rethinking_prompting/blob/HEAD/model.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8686c0a944293a75"}},{"code_sha256_prefix":"cf86a311f5076219","entry":"parse_best_solution","repo":"MraDonkey/rethinking_prompting","repo_kind":"official","path":"model.py","file_url":"https://github.com/MraDonkey/rethinking_prompting/blob/HEAD/model.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cf86a311f5076219"}},{"code_sha256_prefix":"4dd8a18f0e728e70","entry":"LLM_generate","repo":"MraDonkey/rethinking_prompting","repo_kind":"official","path":"model.py","file_url":"https://github.com/MraDonkey/rethinking_prompting/blob/HEAD/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4dd8a18f0e728e70"}},{"code_sha256_prefix":"22f36d100d431b72","entry":"get_messages","repo":"MraDonkey/rethinking_prompting","repo_kind":"official","path":"model.py","file_url":"https://github.com/MraDonkey/rethinking_prompting/blob/HEAD/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"22f36d100d431b72"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}