{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/reasoning-or-reciting-exploring-the","title":"Reasoning or Reciting? Exploring the Capabilities and Limitations of Language Models Through Counterfactual Tasks","arxiv_id":"2307.02477","date":"2023-07-05","proceeding":null,"authors":["Zhaofeng Wu","Linlu Qiu","Alexis Ross","Ekin Akyürek","Boyuan Chen","Bailin Wang","Najoung Kim","Jacob Andreas","Yoon Kim"],"abstract":"The impressive performance of recent language models across a wide range of tasks suggests that they possess a degree of abstract reasoning skills. Are these skills general and transferable, or specialized to specific tasks seen during pretraining? To disentangle these effects, we propose an evaluation framework based on \"counterfactual\" task variants that deviate from the default assumptions underlying standard tasks. Across a suite of 11 tasks, we observe nontrivial performance on the counterfactual variants, but nevertheless find that performance substantially and consistently degrades compared to the default conditions. This suggests that while current LMs may possess abstract task-solving skills to an extent, they often also rely on narrow, non-transferable procedures for task-solving. These results motivate a more careful interpretation of language model performance that teases apart these aspects of behavior.","url_abs":"https://arxiv.org/abs/2307.02477v3","url_pdf":"https://arxiv.org/pdf/2307.02477v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"reasoning-or-reciting-exploring-the","repo_url":"https://github.com/zhaofengwu/counterfactual-evaluation","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":null,"task_name":"counterfactual"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2307.02477","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2307.02477"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/zhaofengwu/counterfactual-evaluation","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":12,"ran_draft_wrong":1,"unverified":2},"by_repo_kind":{"official":{"samples":15,"ran":13,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"dd09dc3fabcd7c46","entry":"encode_images","repo":"zhaofengwu/counterfactual-evaluation","repo_kind":"official","path":"drawing/eval_utils.py","file_url":"https://github.com/zhaofengwu/counterfactual-evaluation/blob/HEAD/drawing/eval_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dd09dc3fabcd7c46"}},{"code_sha256_prefix":"8e2c1bf8f5114da3","entry":"escape","repo":"zhaofengwu/counterfactual-evaluation","repo_kind":"official","path":"arithmetic/query_ccc.py","file_url":"https://github.com/zhaofengwu/counterfactual-evaluation/blob/HEAD/arithmetic/query_ccc.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8e2c1bf8f5114da3"}},{"code_sha256_prefix":"e080db2eaa81fdb3","entry":"estimate_pass_at_k","repo":"zhaofengwu/counterfactual-evaluation","repo_kind":"official","path":"programming/generation/human_eval/evaluation.py","file_url":"https://github.com/zhaofengwu/counterfactual-evaluation/blob/HEAD/programming/generation/human_eval/evaluation.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e080db2eaa81fdb3"}},{"code_sha256_prefix":"4fd941db8b2c2e88","entry":"get_captions","repo":"zhaofengwu/counterfactual-evaluation","repo_kind":"official","path":"drawing/eval_utils.py","file_url":"https://github.com/zhaofengwu/counterfactual-evaluation/blob/HEAD/drawing/eval_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4fd941db8b2c2e88"}},{"code_sha256_prefix":"6cd76a2436afdd9f","entry":"get_label","repo":"zhaofengwu/counterfactual-evaluation","repo_kind":"official","path":"arithmetic/sample.py","file_url":"https://github.com/zhaofengwu/counterfactual-evaluation/blob/HEAD/arithmetic/sample.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6cd76a2436afdd9f"}},{"code_sha256_prefix":"819f8be42e847f42","entry":"load_data","repo":"zhaofengwu/counterfactual-evaluation","repo_kind":"official","path":"chess/query.py","file_url":"https://github.com/zhaofengwu/counterfactual-evaluation/blob/HEAD/chess/query.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"819f8be42e847f42"}},{"code_sha256_prefix":"5352f13bd6bc44b3","entry":"parse_output","repo":"zhaofengwu/counterfactual-evaluation","repo_kind":"official","path":"arithmetic/eval_ccc.py","file_url":"https://github.com/zhaofengwu/counterfactual-evaluation/blob/HEAD/arithmetic/eval_ccc.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5352f13bd6bc44b3"}},{"code_sha256_prefix":"36f593974514e07c","entry":"query_openai","repo":"zhaofengwu/counterfactual-evaluation","repo_kind":"official","path":"query_utils.py","file_url":"https://github.com/zhaofengwu/counterfactual-evaluation/blob/HEAD/query_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"36f593974514e07c"}},{"code_sha256_prefix":"5aa746044ba7f7d5","entry":"sample_number","repo":"zhaofengwu/counterfactual-evaluation","repo_kind":"official","path":"arithmetic/sample.py","file_url":"https://github.com/zhaofengwu/counterfactual-evaluation/blob/HEAD/arithmetic/sample.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5aa746044ba7f7d5"}},{"code_sha256_prefix":"dc7985be0af1dc39","entry":"sample_single","repo":"zhaofengwu/counterfactual-evaluation","repo_kind":"official","path":"arithmetic/sample.py","file_url":"https://github.com/zhaofengwu/counterfactual-evaluation/blob/HEAD/arithmetic/sample.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dc7985be0af1dc39"}},{"code_sha256_prefix":"2c5dcccc856725c9","entry":"templatize","repo":"zhaofengwu/counterfactual-evaluation","repo_kind":"official","path":"arithmetic/query_ccc.py","file_url":"https://github.com/zhaofengwu/counterfactual-evaluation/blob/HEAD/arithmetic/query_ccc.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2c5dcccc856725c9"}},{"code_sha256_prefix":"9639157e1c524625","entry":"templatize","repo":"zhaofengwu/counterfactual-evaluation","repo_kind":"official","path":"chess/query.py","file_url":"https://github.com/zhaofengwu/counterfactual-evaluation/blob/HEAD/chess/query.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9639157e1c524625"}},{"code_sha256_prefix":"274b5d31316b3739","entry":"unescape","repo":"zhaofengwu/counterfactual-evaluation","repo_kind":"official","path":"arithmetic/eval_ccc.py","file_url":"https://github.com/zhaofengwu/counterfactual-evaluation/blob/HEAD/arithmetic/eval_ccc.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"274b5d31316b3739"}},{"code_sha256_prefix":"dd4aafd6d5e91a3c","entry":"encode_classes","repo":"zhaofengwu/counterfactual-evaluation","repo_kind":"official","path":"drawing/eval_utils.py","file_url":"https://github.com/zhaofengwu/counterfactual-evaluation/blob/HEAD/drawing/eval_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dd4aafd6d5e91a3c"}},{"code_sha256_prefix":"19996a3c6b913d67","entry":"load_data","repo":"zhaofengwu/counterfactual-evaluation","repo_kind":"official","path":"arithmetic/query.py","file_url":"https://github.com/zhaofengwu/counterfactual-evaluation/blob/HEAD/arithmetic/query.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"19996a3c6b913d67"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}