{"url":"/sota/common-sense-reasoning-on-commonsenseqa","task":{"name":"Common Sense Reasoning","url":"/task/common-sense-reasoning","note":null},"dataset":{"name":"CommonsenseQA","url":"/dataset/commonsenseqa"},"category":"Natural Language Processing","categories":["Natural Language Processing","Reasoning"],"category_note":null,"description":"Common sense reasoning tasks are intended to require the model to go beyond pattern \nrecognition. Instead, the model should use \"common sense\" or world knowledge\nto make inferences.","description_from":"task","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","rank":"the archive's row order at snapshot; not re-ranked","rows_end_at":"2025-07-28","rows_withheld_as_spam":0,"metric_values":"the archive's strings, untouched"},"metrics":["Accuracy"],"metric_direction":{"note":"inferred from the metric name only (the archive records no direction); null = not inferred, chart draws points only","by_metric":{"Accuracy":"higher"}},"counts":{"rows":38,"rows_with_code":35,"rows_with_paper_page":38,"rows_dated":38,"rows_using_additional_data":8},"rows":[{"rank_in_archive_order":1,"model":"GPT-4o (HPT)","metrics":{"Accuracy":"92.54"},"uses_additional_data":false,"paper_date":"2024-06-18","paper":"/paper/hierarchical-prompting-taxonomy-a-universal","paper_url":"https://arxiv.org/abs/2406.12644v4","paper_title":"Hierarchical Prompting Taxonomy: A Universal Evaluation Framework for Large Language Models Aligned with Human Cognitive Principles","code":"https://github.com/devichand579/HPT","n_code_links":1,"syntology":null},{"rank_in_archive_order":2,"model":"DeBERTaV3-large+KEAR","metrics":{"Accuracy":"91.2"},"uses_additional_data":true,"paper_date":"2021-12-06","paper":"/paper/human-parity-on-commonsenseqa-augmenting-self","paper_url":"https://arxiv.org/abs/2112.03254v3","paper_title":"Human Parity on CommonsenseQA: Augmenting Self-Attention with External Attention","code":"https://github.com/microsoft/DEKCOR-CommonsenseQA","n_code_links":2,"syntology":null},{"rank_in_archive_order":3,"model":"PaLM 2 (few‑shot, CoT, SC)","metrics":{"Accuracy":"90.4"},"uses_additional_data":true,"paper_date":"2023-05-17","paper":"/paper/palm-2-technical-report-1","paper_url":"https://arxiv.org/abs/2305.10403v3","paper_title":"PaLM 2 Technical Report","code":"https://github.com/eternityyw/tram-benchmark","n_code_links":1,"syntology":null},{"rank_in_archive_order":4,"model":"KEAR","metrics":{"Accuracy":"89.4"},"uses_additional_data":true,"paper_date":"2021-12-06","paper":"/paper/human-parity-on-commonsenseqa-augmenting-self","paper_url":"https://arxiv.org/abs/2112.03254v3","paper_title":"Human Parity on CommonsenseQA: Augmenting Self-Attention with External Attention","code":"https://github.com/microsoft/DEKCOR-CommonsenseQA","n_code_links":2,"syntology":null},{"rank_in_archive_order":5,"model":"DEKCOR","metrics":{"Accuracy":"83.3"},"uses_additional_data":true,"paper_date":"2020-12-09","paper":"/paper/fusing-context-into-knowledge-graph-for","paper_url":"https://arxiv.org/abs/2012.04808v3","paper_title":"Fusing Context Into Knowledge Graph for Commonsense Question Answering","code":"https://github.com/microsoft/kear","n_code_links":2,"syntology":null},{"rank_in_archive_order":6,"model":"Unicorn 11B (fine-tuned)","metrics":{"Accuracy":"79.3"},"uses_additional_data":false,"paper_date":"2021-03-24","paper":"/paper/unicorn-on-rainbow-a-universal-commonsense","paper_url":"https://arxiv.org/abs/2103.13009v1","paper_title":"UNICORN on RAINBOW: A Universal Commonsense Reasoning Model on a New Multitask Benchmark","code":"https://github.com/allenai/rainbow","n_code_links":1,"syntology":{"n_ran":0,"n_unverified":5,"n_samples":5,"n_pointer_only_licence":0}},{"rank_in_archive_order":7,"model":"MUPPET Roberta Large","metrics":{"Accuracy":"79.2"},"uses_additional_data":true,"paper_date":"2021-01-26","paper":"/paper/muppet-massive-multi-task-representations","paper_url":"https://arxiv.org/abs/2101.11038v1","paper_title":"Muppet: Massive Multi-task Representations with Pre-Finetuning","code":"https://huggingface.co/facebook/muppet-roberta-base","n_code_links":2,"syntology":null},{"rank_in_archive_order":8,"model":"UnifiedQA 11B (fine-tuned)","metrics":{"Accuracy":"79.1"},"uses_additional_data":true,"paper_date":"2020-05-02","paper":"/paper/unifiedqa-crossing-format-boundaries-with-a","paper_url":"https://arxiv.org/abs/2005.00700v3","paper_title":"UnifiedQA: Crossing Format Boundaries With a Single QA System","code":"https://github.com/allenai/unifiedqa","n_code_links":2,"syntology":{"n_ran":4,"n_unverified":3,"n_samples":7,"n_pointer_only_licence":1}},{"rank_in_archive_order":9,"model":"DRAGON","metrics":{"Accuracy":"78.2"},"uses_additional_data":false,"paper_date":"2022-10-17","paper":"/paper/deep-bidirectional-language-knowledge-graph","paper_url":"https://arxiv.org/abs/2210.09338v2","paper_title":"Deep Bidirectional Language-Knowledge Graph Pretraining","code":"https://github.com/michiyasunaga/dragon","n_code_links":2,"syntology":{"n_ran":2,"n_unverified":16,"n_samples":18,"n_pointer_only_licence":0}},{"rank_in_archive_order":10,"model":"T5-XXL 11B (fine-tuned)","metrics":{"Accuracy":"78.1"},"uses_additional_data":false,"paper_date":"2020-05-02","paper":"/paper/unifiedqa-crossing-format-boundaries-with-a","paper_url":"https://arxiv.org/abs/2005.00700v3","paper_title":"UnifiedQA: Crossing Format Boundaries With a Single QA System","code":"https://github.com/allenai/unifiedqa","n_code_links":2,"syntology":{"n_ran":4,"n_unverified":3,"n_samples":7,"n_pointer_only_licence":1}},{"rank_in_archive_order":11,"model":"Albert Lan et al. (2020) (ensemble)","metrics":{"Accuracy":"76.5"},"uses_additional_data":false,"paper_date":"2019-09-26","paper":"/paper/albert-a-lite-bert-for-self-supervised","paper_url":"https://arxiv.org/abs/1909.11942v6","paper_title":"ALBERT: A Lite BERT for Self-supervised Learning of Language Representations","code":"https://github.com/huggingface/transformers","n_code_links":48,"syntology":{"n_ran":46,"n_unverified":80,"n_samples":126,"n_pointer_only_licence":22}},{"rank_in_archive_order":12,"model":"UnifiedQA 11B (zero-shot)","metrics":{"Accuracy":"76.2"},"uses_additional_data":false,"paper_date":"2020-05-02","paper":"/paper/unifiedqa-crossing-format-boundaries-with-a","paper_url":"https://arxiv.org/abs/2005.00700v3","paper_title":"UnifiedQA: Crossing Format Boundaries With a Single QA System","code":"https://github.com/allenai/unifiedqa","n_code_links":2,"syntology":{"n_ran":4,"n_unverified":3,"n_samples":7,"n_pointer_only_licence":1}},{"rank_in_archive_order":13,"model":"QA-GNN","metrics":{"Accuracy":"76.1"},"uses_additional_data":false,"paper_date":"2021-04-13","paper":"/paper/qa-gnn-reasoning-with-language-models-and","paper_url":"https://arxiv.org/abs/2104.06378v5","paper_title":"QA-GNN: Reasoning with Language Models and Knowledge Graphs for Question Answering","code":"https://github.com/michiyasunaga/qagnn","n_code_links":6,"syntology":{"n_ran":1,"n_unverified":24,"n_samples":25,"n_pointer_only_licence":4}},{"rank_in_archive_order":14,"model":"XLNet+GraphReason","metrics":{"Accuracy":"75.3"},"uses_additional_data":false,"paper_date":"2019-09-09","paper":"/paper/graph-based-reasoning-over-heterogeneous","paper_url":"https://arxiv.org/abs/1909.05311v2","paper_title":"Graph-Based Reasoning over Heterogeneous External Knowledge for Commonsense Question Answering","code":"https://github.com/DecstionBack/AAAI_2020_CommonsenseQA","n_code_links":1,"syntology":{"n_ran":7,"n_unverified":10,"n_samples":17,"n_pointer_only_licence":0}},{"rank_in_archive_order":15,"model":"GrapeQA: PEGA","metrics":{"Accuracy":"73.5"},"uses_additional_data":false,"paper_date":"2023-03-22","paper":"/paper/grapeqa-graph-augmentation-and-pruning-to","paper_url":"https://arxiv.org/abs/2303.12320v2","paper_title":"GrapeQA: GRaph Augmentation and Pruning to Enhance Question-Answering","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":16,"model":"RoBERTa+HyKAS Ma et al. (2019)","metrics":{"Accuracy":"73.2"},"uses_additional_data":false,"paper_date":"2019-10-30","paper":"/paper/towards-generalizable-neuro-symbolic-systems","paper_url":"https://arxiv.org/abs/1910.14087v1","paper_title":"Towards Generalizable Neuro-Symbolic Systems for Commonsense Question Answering","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":17,"model":"GPT-3 Direct Finetuned","metrics":{"Accuracy":"73.0"},"uses_additional_data":false,"paper_date":"2021-12-06","paper":"/paper/human-parity-on-commonsenseqa-augmenting-self","paper_url":"https://arxiv.org/abs/2112.03254v3","paper_title":"Human Parity on CommonsenseQA: Augmenting Self-Attention with External Attention","code":"https://github.com/microsoft/DEKCOR-CommonsenseQA","n_code_links":2,"syntology":null},{"rank_in_archive_order":18,"model":"STaR (on GPT-J)","metrics":{"Accuracy":"72.3"},"uses_additional_data":false,"paper_date":"2022-03-28","paper":"/paper/star-bootstrapping-reasoning-with-reasoning","paper_url":"https://arxiv.org/abs/2203.14465v2","paper_title":"STaR: Bootstrapping Reasoning With Reasoning","code":"https://github.com/ezelikman/STaR","n_code_links":1,"syntology":{"n_ran":0,"n_unverified":2,"n_samples":2,"n_pointer_only_licence":0}},{"rank_in_archive_order":19,"model":"RoBERTa-Large 355M","metrics":{"Accuracy":"72.1"},"uses_additional_data":false,"paper_date":"2019-07-26","paper":"/paper/roberta-a-robustly-optimized-bert-pretraining","paper_url":"https://arxiv.org/abs/1907.11692v1","paper_title":"RoBERTa: A Robustly Optimized BERT Pretraining Approach","code":"https://github.com/huggingface/transformers","n_code_links":67,"syntology":{"n_ran":22,"n_unverified":26,"n_samples":48,"n_pointer_only_licence":23}},{"rank_in_archive_order":20,"model":"STaR without Rationalization (on GPT-J)","metrics":{"Accuracy":"68.8"},"uses_additional_data":false,"paper_date":"2022-03-28","paper":"/paper/star-bootstrapping-reasoning-with-reasoning","paper_url":"https://arxiv.org/abs/2203.14465v2","paper_title":"STaR: Bootstrapping Reasoning With Reasoning","code":"https://github.com/ezelikman/STaR","n_code_links":1,"syntology":{"n_ran":0,"n_unverified":2,"n_samples":2,"n_pointer_only_licence":0}},{"rank_in_archive_order":21,"model":"OPT 66B (1-shot)","metrics":{"Accuracy":"66.4"},"uses_additional_data":false,"paper_date":"2023-03-30","paper":"/paper/bloomberggpt-a-large-language-model-for","paper_url":"https://arxiv.org/abs/2303.17564v3","paper_title":"BloombergGPT: A Large Language Model for Finance","code":"https://github.com/yangletliu/finlora","n_code_links":2,"syntology":null},{"rank_in_archive_order":22,"model":"Bloomberg GPT 50B (1-shot)","metrics":{"Accuracy":"65.5"},"uses_additional_data":false,"paper_date":"2023-03-30","paper":"/paper/bloomberggpt-a-large-language-model-for","paper_url":"https://arxiv.org/abs/2303.17564v3","paper_title":"BloombergGPT: A Large Language Model for Finance","code":"https://github.com/yangletliu/finlora","n_code_links":2,"syntology":null},{"rank_in_archive_order":23,"model":"CAGE-reasoning","metrics":{"Accuracy":"64.7"},"uses_additional_data":false,"paper_date":"2019-06-06","paper":"/paper/explain-yourself-leveraging-language-models","paper_url":"https://arxiv.org/abs/1906.02361v1","paper_title":"Explain Yourself! Leveraging Language Models for Commonsense Reasoning","code":"https://github.com/salesforce/cos-e","n_code_links":1,"syntology":{"n_ran":5,"n_unverified":0,"n_samples":5,"n_pointer_only_licence":1}},{"rank_in_archive_order":24,"model":"BLOOM 176B (1-shot)","metrics":{"Accuracy":"64.2"},"uses_additional_data":false,"paper_date":"2023-03-30","paper":"/paper/bloomberggpt-a-large-language-model-for","paper_url":"https://arxiv.org/abs/2303.17564v3","paper_title":"BloombergGPT: A Large Language Model for Finance","code":"https://github.com/yangletliu/finlora","n_code_links":2,"syntology":null},{"rank_in_archive_order":25,"model":"UnifiedQA 440M (fine-tuned)","metrics":{"Accuracy":"64"},"uses_additional_data":false,"paper_date":"2020-05-02","paper":"/paper/unifiedqa-crossing-format-boundaries-with-a","paper_url":"https://arxiv.org/abs/2005.00700v3","paper_title":"UnifiedQA: Crossing Format Boundaries With a Single QA System","code":"https://github.com/allenai/unifiedqa","n_code_links":2,"syntology":{"n_ran":4,"n_unverified":3,"n_samples":7,"n_pointer_only_licence":1}},{"rank_in_archive_order":26,"model":"BART-large 440M (fine-tuned)","metrics":{"Accuracy":"62.5"},"uses_additional_data":false,"paper_date":"2020-05-02","paper":"/paper/unifiedqa-crossing-format-boundaries-with-a","paper_url":"https://arxiv.org/abs/2005.00700v3","paper_title":"UnifiedQA: Crossing Format Boundaries With a Single QA System","code":"https://github.com/allenai/unifiedqa","n_code_links":2,"syntology":{"n_ran":4,"n_unverified":3,"n_samples":7,"n_pointer_only_licence":1}},{"rank_in_archive_order":27,"model":"BERT_CSlarge","metrics":{"Accuracy":"62.2"},"uses_additional_data":false,"paper_date":"2019-08-19","paper":"/paper/align-mask-and-select-a-simple-method-for","paper_url":"https://arxiv.org/abs/1908.06725v5","paper_title":"Align, Mask and Select: A Simple Method for Incorporating Commonsense Knowledge into Language Representation Models","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":28,"model":"GPT-NeoX 20B (1-shot)","metrics":{"Accuracy":"60.4"},"uses_additional_data":false,"paper_date":"2023-03-30","paper":"/paper/bloomberggpt-a-large-language-model-for","paper_url":"https://arxiv.org/abs/2303.17564v3","paper_title":"BloombergGPT: A Large Language Model for Finance","code":"https://github.com/yangletliu/finlora","n_code_links":2,"syntology":null},{"rank_in_archive_order":29,"model":"GPT-J Direct Finetuned","metrics":{"Accuracy":"60.0"},"uses_additional_data":false,"paper_date":"2022-03-28","paper":"/paper/star-bootstrapping-reasoning-with-reasoning","paper_url":"https://arxiv.org/abs/2203.14465v2","paper_title":"STaR: Bootstrapping Reasoning With Reasoning","code":"https://github.com/ezelikman/STaR","n_code_links":1,"syntology":{"n_ran":0,"n_unverified":2,"n_samples":2,"n_pointer_only_licence":0}},{"rank_in_archive_order":30,"model":"KagNet","metrics":{"Accuracy":"58.9"},"uses_additional_data":true,"paper_date":"2019-09-04","paper":"/paper/kagnet-knowledge-aware-graph-networks-for","paper_url":"https://arxiv.org/abs/1909.02151v1","paper_title":"KagNet: Knowledge-Aware Graph Networks for Commonsense Reasoning","code":"https://github.com/INK-USC/KagNet","n_code_links":2,"syntology":{"n_ran":1,"n_unverified":11,"n_samples":12,"n_pointer_only_licence":0}},{"rank_in_archive_order":31,"model":"BERT-LARGE","metrics":{"Accuracy":"55.9"},"uses_additional_data":true,"paper_date":"2018-11-02","paper":"/paper/commonsenseqa-a-question-answering-challenge","paper_url":"http://arxiv.org/abs/1811.00937v2","paper_title":"CommonsenseQA: A Question Answering Challenge Targeting Commonsense Knowledge","code":"https://github.com/jonathanherzig/commonsenseqa","n_code_links":4,"syntology":{"n_ran":0,"n_unverified":3,"n_samples":3,"n_pointer_only_licence":0}},{"rank_in_archive_order":32,"model":"UL2 20B (chain-of-thought + self-consistency)","metrics":{"Accuracy":"55.7"},"uses_additional_data":false,"paper_date":"2022-05-10","paper":"/paper/unifying-language-learning-paradigms","paper_url":"https://arxiv.org/abs/2205.05131v3","paper_title":"UL2: Unifying Language Learning Paradigms","code":"https://github.com/google-research/google-research","n_code_links":2,"syntology":{"n_ran":0,"n_unverified":16,"n_samples":16,"n_pointer_only_licence":0}},{"rank_in_archive_order":33,"model":"Few-shot CoT LaMDA 137B","metrics":{"Accuracy":" 55.6"},"uses_additional_data":false,"paper_date":"2022-03-28","paper":"/paper/star-bootstrapping-reasoning-with-reasoning","paper_url":"https://arxiv.org/abs/2203.14465v2","paper_title":"STaR: Bootstrapping Reasoning With Reasoning","code":"https://github.com/ezelikman/STaR","n_code_links":1,"syntology":{"n_ran":0,"n_unverified":2,"n_samples":2,"n_pointer_only_licence":0}},{"rank_in_archive_order":34,"model":"UL2 20B (chain-of-thought)","metrics":{"Accuracy":"51.4"},"uses_additional_data":false,"paper_date":"2022-05-10","paper":"/paper/unifying-language-learning-paradigms","paper_url":"https://arxiv.org/abs/2205.05131v3","paper_title":"UL2: Unifying Language Learning Paradigms","code":"https://github.com/google-research/google-research","n_code_links":2,"syntology":{"n_ran":0,"n_unverified":16,"n_samples":16,"n_pointer_only_licence":0}},{"rank_in_archive_order":35,"model":"Few-shot CoT GPT-J","metrics":{"Accuracy":"36.6"},"uses_additional_data":false,"paper_date":"2022-03-28","paper":"/paper/star-bootstrapping-reasoning-with-reasoning","paper_url":"https://arxiv.org/abs/2203.14465v2","paper_title":"STaR: Bootstrapping Reasoning With Reasoning","code":"https://github.com/ezelikman/STaR","n_code_links":1,"syntology":{"n_ran":0,"n_unverified":2,"n_samples":2,"n_pointer_only_licence":0}},{"rank_in_archive_order":36,"model":"UL2 20B (zero-shot)","metrics":{"Accuracy":"34.2"},"uses_additional_data":false,"paper_date":"2022-05-10","paper":"/paper/unifying-language-learning-paradigms","paper_url":"https://arxiv.org/abs/2205.05131v3","paper_title":"UL2: Unifying Language Learning Paradigms","code":"https://github.com/google-research/google-research","n_code_links":2,"syntology":{"n_ran":0,"n_unverified":16,"n_samples":16,"n_pointer_only_licence":0}},{"rank_in_archive_order":37,"model":"Chain of thought ASDiv","metrics":{"Accuracy":"28.6"},"uses_additional_data":false,"paper_date":"2022-01-28","paper":"/paper/chain-of-thought-prompting-elicits-reasoning","paper_url":"https://arxiv.org/abs/2201.11903v6","paper_title":"Chain-of-Thought Prompting Elicits Reasoning in Large Language Models","code":"https://github.com/microsoft/guidance","n_code_links":19,"syntology":{"n_ran":2,"n_unverified":5,"n_samples":7,"n_pointer_only_licence":0}},{"rank_in_archive_order":38,"model":"Few-shot Direct GPT-J","metrics":{"Accuracy":"20.9"},"uses_additional_data":false,"paper_date":"2022-03-28","paper":"/paper/star-bootstrapping-reasoning-with-reasoning","paper_url":"https://arxiv.org/abs/2203.14465v2","paper_title":"STaR: Bootstrapping Reasoning With Reasoning","code":"https://github.com/ezelikman/STaR","n_code_links":1,"syntology":{"n_ran":0,"n_unverified":2,"n_samples":2,"n_pointer_only_licence":0}}],"since_archive":{"claim":"Results that newer papers report for their own method, placed here by Syntology. A model pointed at the cell in the paper's own table; the number was read from that cell and checked against this leaderboard's metric, dataset, split and scale; an independent check that saw this leaderboard's other rows and every other leaderboard on the same dataset accepted it. Not reviewed by the paper's authors or by the archive's editors, and not ranked against the archive rows.","extraction_file_present":true,"measurement":{"test_papers":883,"papers_with_output":881,"judged_true":108,"judged":110,"wilson95_lower":0.9361,"measured_on":"2026-09-24","frozen_commit":"0e3de0df94"},"measurement_note":"blind adjudication of accepted entries on a held-out split of archive papers, rules frozen before the test","coverage":{"sentence":"Syntology has checked 6,264 of the 9,581 papers on this site that are newer than the archive; results from the others appear after they are checked.","complete":false,"papers_newer_than_archive":9581,"papers_checked":6264,"papers_extracted_not_yet_verified":0,"boards_without_verdict":2,"papers_not_yet_extracted":3316},"order":"newest first by month (arXiv date, else the arXiv-id month), then arXiv id descending","columns":["Accuracy"],"entries":[{"paper":"/paper/arxiv-2607-17266","paper_title":"Debate-on-Graph: Reliable and Adaptive Reasoning of Large Language Model on Uncertain Knowledge Graph","arxiv_id":"2607.17266","model":"DoG (Ours) · GPT","row_label":"DoG (Ours)","configuration":"GPT","configuration_in_model":true,"configuration_not_shown_because":null,"values":{"Accuracy":"86.0"},"date":"2026-07-19","month_from_arxiv_id":null,"where_in_paper":"Table 2, row “DoG (Ours)”","table_label":"2","table_label_from":"the caption's own label","table_position_1based":2,"caption_head":"Table 2 : Accuracy (%) comparison of DoG and DoG-E with different baselines on…","cells":[{"column":"Accuracy","cell_text":"86.0","column_header":"CSQA / GPT","value":86.0,"table_index_0based":1,"row":13,"col":2}],"code":"3 of 3 ran"}]},"syntology":{"read_at":"2026-09-24T18:15:14+00:00","claim":"Per row: N of M harvested code samples from that row's paper executed on a synthesized fixture; the other M-N are unverified. Not a reproduction of the row's number; not a correctness claim. n_pointer_only_licence counts samples the site points at rather than redistributes (a licence axis, independent of ran/unverified).","rows_with_graph_line":24,"rows_with_any_sample_ran":13,"distinct_papers_with_graph_line":13,"distinct_papers_with_any_sample_ran":9,"samples_over_distinct_papers":{"n_ran":90,"n_unverified":201,"n_samples":291,"n_pointer_only_licence":51,"note":"each paper (arXiv id) counted once, however many rows it is behind; this is the page-level figure"},"samples_row_weighted":{"n_ran":106,"n_unverified":255,"n_samples":361,"n_pointer_only_licence":55,"note":"row-weighted: a paper behind several rows is counted once per row; inflated relative to samples_over_distinct_papers by design, kept for readers summing the per-row syntology blocks"}}}