{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2510-22849","title":"Once Upon an Input: Reasoning via Per-Instance Program Synthesis","arxiv_id":"2510.22849","date":"2025-10-26","proceeding":"NeurIPS","authors":["Adam Stein","Neelay Velingker","Mayur Naik","Eric Wong"],"abstract":"Large language models (LLMs) excel at zero-shot inference but continue to struggle with complex, multi-step reasoning. Recent methods that augment LLMs with intermediate reasoning steps such as Chain of Thought (CoT) and Program of Thought (PoT) improve performance but often produce undesirable solutions, especially in algorithmic domains. We introduce Per-Instance Program Synthesis (PIPS), a method that generates and refines programs at the instance-level using structural feedback without relying on task-specific guidance or explicit test cases. To further improve performance, PIPS incorporates a confidence metric that dynamically chooses between direct inference and program synthesis on a per-instance basis. Experiments across three frontier LLMs and 30 benchmarks including all tasks of Big Bench Extra Hard (BBEH), visual question answering tasks, relational reasoning tasks, and mathematical reasoning tasks show that PIPS improves the absolute harmonic mean accuracy by up to 8.6% and 9.4% compared to PoT and CoT respectively, and reduces undesirable program generations by 65.1% on the algorithmic tasks compared to PoT with Gemini-2.0-Flash.","url_abs":"https://arxiv.org/abs/2510.22849","url_pdf":"https://arxiv.org/pdf/2510.22849","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2510.22849","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2510.22849"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/adaminsky/pips","reach":null}],"summary":{"ran":4,"ran_draft_wrong":3,"unverified":1},"by_repo_kind":{"found_in_text":{"samples":8,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"282a3f570d2ec2e1","entry":"LLMModel","repo":"adaminsky/pips","repo_kind":"found_in_text","path":"src/pips/core.py","file_url":"https://github.com/adaminsky/pips/blob/HEAD/src/pips/core.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"282a3f570d2ec2e1"}},{"code_sha256_prefix":"50d9e1b62e745597","entry":"OpenAIModel","repo":"adaminsky/pips","repo_kind":"found_in_text","path":"src/pips/core.py","file_url":"https://github.com/adaminsky/pips/blob/HEAD/src/pips/core.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"50d9e1b62e745597"}},{"code_sha256_prefix":"f2c47b3ebe7a993b","entry":"RawInput","repo":"adaminsky/pips","repo_kind":"found_in_text","path":"src/pips/core.py","file_url":"https://github.com/adaminsky/pips/blob/HEAD/src/pips/core.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f2c47b3ebe7a993b"}},{"code_sha256_prefix":"b532f5885353756d","entry":"SamplingParams","repo":"adaminsky/pips","repo_kind":"found_in_text","path":"src/pips/core.py","file_url":"https://github.com/adaminsky/pips/blob/HEAD/src/pips/core.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b532f5885353756d"}},{"code_sha256_prefix":"1f829c4cbf667fdc","entry":"img2base64","repo":"adaminsky/pips","repo_kind":"found_in_text","path":"src/pips/core.py","file_url":"https://github.com/adaminsky/pips/blob/HEAD/src/pips/core.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1f829c4cbf667fdc"}},{"code_sha256_prefix":"02d2f1dfe6746ca8","entry":"python_eval","repo":"adaminsky/pips","repo_kind":"found_in_text","path":"src/pips/core.py","file_url":"https://github.com/adaminsky/pips/blob/HEAD/src/pips/core.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"02d2f1dfe6746ca8"}},{"code_sha256_prefix":"b3efff0266ee76d0","entry":"run_with_timeout","repo":"adaminsky/pips","repo_kind":"found_in_text","path":"src/pips/core.py","file_url":"https://github.com/adaminsky/pips/blob/HEAD/src/pips/core.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b3efff0266ee76d0"}},{"code_sha256_prefix":"a2e87ede6b1a1878","entry":"PIPSSolver","repo":"adaminsky/pips","repo_kind":"found_in_text","path":"src/pips/core.py","file_url":"https://github.com/adaminsky/pips/blob/HEAD/src/pips/core.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a2e87ede6b1a1878"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.CL","source":"arxiv_api"},"syntology_extracted_results":null}