{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/cogbench-a-large-language-model-walks-into-a","title":"CogBench: a large language model walks into a psychology lab","arxiv_id":"2402.18225","date":"2024-02-28","proceeding":null,"authors":["Julian Coda-Forno","Marcel Binz","Jane X. Wang","Eric Schulz"],"abstract":"Large language models (LLMs) have significantly advanced the field of artificial intelligence. Yet, evaluating them comprehensively remains challenging. We argue that this is partly due to the predominant focus on performance metrics in most benchmarks. This paper introduces CogBench, a benchmark that includes ten behavioral metrics derived from seven cognitive psychology experiments. This novel approach offers a toolkit for phenotyping LLMs' behavior. We apply CogBench to 35 LLMs, yielding a rich and diverse dataset. We analyze this data using statistical multilevel modeling techniques, accounting for the nested dependencies among fine-tuned versions of specific LLMs. Our study highlights the crucial role of model size and reinforcement learning from human feedback (RLHF) in improving performance and aligning with human behavior. Interestingly, we find that open-source models are less risk-prone than proprietary models and that fine-tuning on code does not necessarily enhance LLMs' behavior. Finally, we explore the effects of prompt-engineering techniques. We discover that chain-of-thought prompting improves probabilistic reasoning, while take-a-step-back prompting fosters model-based behaviors.","url_abs":"https://arxiv.org/abs/2402.18225v1","url_pdf":"https://arxiv.org/pdf/2402.18225v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"cogbench-a-large-language-model-walks-into-a","repo_url":"https://github.com/juliancodaforno/cogbench","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"large-language-model","task_name":"Large Language Model"},{"task_slug":"prompt-engineering","task_name":"Prompt Engineering"}],"methods":[{"method_slug":"focus","method_name":"Focus"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2402.18225","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.18225"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/juliancodaforno/cogbench","reach":{"status":"ok"}}],"summary":{"ran":6,"unverified":2},"by_repo_kind":{"official":{"samples":8,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":8,"samples":[{"code_sha256_prefix":"a6afa557dfeece3a","entry":"aggregated_results","repo":"juliancodaforno/cogbench","repo_kind":"official","path":"Analysis/utils.py","file_url":"https://github.com/juliancodaforno/cogbench/blob/HEAD/Analysis/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a6afa557dfeece3a"}},{"code_sha256_prefix":"fec1cdde9a74bf06","entry":"filter_models","repo":"juliancodaforno/cogbench","repo_kind":"official","path":"Analysis/phenotype_comp.py","file_url":"https://github.com/juliancodaforno/cogbench/blob/HEAD/Analysis/phenotype_comp.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fec1cdde9a74bf06"}},{"code_sha256_prefix":"957251572ec93244","entry":"get_means_and_cis","repo":"juliancodaforno/cogbench","repo_kind":"official","path":"Analysis/utils.py","file_url":"https://github.com/juliancodaforno/cogbench/blob/HEAD/Analysis/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"957251572ec93244"}},{"code_sha256_prefix":"0c125e1fc7c2994b","entry":"p_to_stars","repo":"juliancodaforno/cogbench","repo_kind":"official","path":"Analysis/utils.py","file_url":"https://github.com/juliancodaforno/cogbench/blob/HEAD/Analysis/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0c125e1fc7c2994b"}},{"code_sha256_prefix":"2f6edf98d2841882","entry":"remove_excessive_newlines","repo":"juliancodaforno/cogbench","repo_kind":"official","path":"utils.py","file_url":"https://github.com/juliancodaforno/cogbench/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2f6edf98d2841882"}},{"code_sha256_prefix":"669ba90f4cef0672","entry":"remove_repeated_sentences","repo":"juliancodaforno/cogbench","repo_kind":"official","path":"utils.py","file_url":"https://github.com/juliancodaforno/cogbench/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"669ba90f4cef0672"}},{"code_sha256_prefix":"28768b3b00640788","entry":"normalize_metrics","repo":"juliancodaforno/cogbench","repo_kind":"official","path":"Analysis/phenotype_comp.py","file_url":"https://github.com/juliancodaforno/cogbench/blob/HEAD/Analysis/phenotype_comp.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"28768b3b00640788"}},{"code_sha256_prefix":"28f0e542b41a578f","entry":"prepare_data_for_plotting","repo":"juliancodaforno/cogbench","repo_kind":"official","path":"Analysis/phenotype_comp.py","file_url":"https://github.com/juliancodaforno/cogbench/blob/HEAD/Analysis/phenotype_comp.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"28f0e542b41a578f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}