{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/commonsenseqa-a-question-answering-challenge","title":"CommonsenseQA: A Question Answering Challenge Targeting Commonsense Knowledge","arxiv_id":"1811.00937","date":"2018-11-02","proceeding":"NAACL 2019 6","authors":["Alon Talmor","Jonathan Herzig","Nicholas Lourie","Jonathan Berant"],"abstract":"When answering a question, people often draw upon their rich world knowledge\nin addition to the particular context. Recent work has focused primarily on\nanswering questions given some relevant document or context, and required very\nlittle general background. To investigate question answering with prior\nknowledge, we present CommonsenseQA: a challenging new dataset for commonsense\nquestion answering. To capture common sense beyond associations, we extract\nfrom ConceptNet (Speer et al., 2017) multiple target concepts that have the\nsame semantic relation to a single source concept. Crowd-workers are asked to\nauthor multiple-choice questions that mention the source concept and\ndiscriminate in turn between each of the target concepts. This encourages\nworkers to create questions with complex semantics that often require prior\nknowledge. We create 12,247 questions through this procedure and demonstrate\nthe difficulty of our task with a large number of strong baselines. Our best\nbaseline is based on BERT-large (Devlin et al., 2018) and obtains 56% accuracy,\nwell below human performance, which is 89%.","url_abs":"http://arxiv.org/abs/1811.00937v2","url_pdf":"http://arxiv.org/pdf/1811.00937v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"commonsenseqa-a-question-answering-challenge","repo_url":"https://github.com/jonathanherzig/commonsenseqa","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"commonsenseqa-a-question-answering-challenge","repo_url":"https://github.com/francois-rd/accord","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"commonsenseqa-a-question-answering-challenge","repo_url":"https://github.com/hkunlp/batch-prompting","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"commonsenseqa-a-question-answering-challenge","repo_url":"https://github.com/xlang-ai/batch-prompting","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}}],"tasks":[{"task_slug":"common-sense-reasoning","task_name":"Common Sense Reasoning"},{"task_slug":"multiple-choice","task_name":"Multiple-choice"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"world-knowledge","task_name":"World Knowledge"}],"methods":[],"datasets_introduced":[{"slug":"commonsenseqa","name":"CommonsenseQA","full_name":"CSQA"}],"methods_introduced":[],"results":[{"leaderboard":"/sota/common-sense-reasoning-on-commonsenseqa","task":"Common Sense Reasoning","dataset":"CommonsenseQA","model":"BERT-LARGE","rank_in_archive_order":31,"of":38,"metrics":{"Accuracy":"55.9"},"uses_additional_data":true}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1811.00937","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1811.00937"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/francois-rd/accord","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/xlang-ai/batch-prompting","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jonathanherzig/commonsenseqa","reach":{"status":"ok"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/google-research/bert","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/hkunlp/batch-prompting","reach":{"status":"ok"}}],"summary":{"unverified":3},"by_repo_kind":{"listed":{"samples":3,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"ab40a5e20cc1408e","entry":"ensure_path","repo":"francois-rd/accord","repo_kind":"listed","path":"src/accord/io.py","file_url":"https://github.com/francois-rd/accord/blob/HEAD/src/accord/io.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ab40a5e20cc1408e"}},{"code_sha256_prefix":"17d2bfe0a0e4a415","entry":"enum_dict_factory","repo":"francois-rd/accord","repo_kind":"listed","path":"src/accord/io.py","file_url":"https://github.com/francois-rd/accord/blob/HEAD/src/accord/io.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"17d2bfe0a0e4a415"}},{"code_sha256_prefix":"f4156c6f7f4efd69","entry":"insert_text","repo":"francois-rd/accord","repo_kind":"listed","path":"src/accord/io.py","file_url":"https://github.com/francois-rd/accord/blob/HEAD/src/accord/io.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f4156c6f7f4efd69"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}