{"url":"/task/common-sense-reasoning","name":"Common Sense Reasoning","slug":"common-sense-reasoning","description_markdown":"Common sense reasoning tasks are intended to require the model to go beyond pattern \nrecognition. Instead, the model should use \"common sense\" or world knowledge\nto make inferences.","categories":[{"name":"Natural Language Processing","url":"/area/natural-language-processing"},{"name":"Reasoning","url":"/area/reasoning"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":939,"papers_with_code":325,"benchmarks":24,"benchmark_tables_in_archive":24,"benchmark_tables_shown":24,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":56,"subtasks":17,"parent_tasks":0},"benchmarks":[{"leaderboard":"/sota/common-sense-reasoning-on-winogrande","slug":"common-sense-reasoning-on-winogrande","dataset":"WinoGrande","dataset_url":"/dataset/winogrande","rows_in_archive":77,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"ST-MoE-32B 269B (fine-tuned)","paper_title":"ST-MoE: Designing Stable and Transferable Sparse Expert Models","paper_url":"/paper/designing-effective-sparse-expert-models","paper_date":"2022-02-17","arxiv_id":"2202.08906","code_links":[{"title":"tensorflow/mesh","url":"https://github.com/tensorflow/mesh"},{"title":"xuefuzhao/openmoe","url":"https://github.com/xuefuzhao/openmoe"},{"title":"yikangshen/megablocks","url":"https://github.com/yikangshen/megablocks"}],"syntology":{"n":5,"n_ran":5,"n_unverified":0,"n_pointer_only":5}}},{"leaderboard":"/sota/common-sense-reasoning-on-arc-challenge","slug":"common-sense-reasoning-on-arc-challenge","dataset":"ARC (Challenge)","dataset_url":"/dataset/arc","rows_in_archive":54,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"GPT-4 (few-shot, k=25)","paper_title":"GPT-4 Technical Report","paper_url":"/paper/gpt-4-technical-report-1","paper_date":"2023-03-15","arxiv_id":"2303.08774","code_links":[{"title":"openai/evals","url":"https://github.com/openai/evals"},{"title":"shmsw25/factscore","url":"https://github.com/shmsw25/factscore"},{"title":"unispac/visual-adversarial-examples-jailbreak-large-language-models","url":"https://github.com/unispac/visual-adversarial-examples-jailbreak-large-language-models"},{"title":"gpt4life/alpagasus","url":"https://github.com/gpt4life/alpagasus"},{"title":"emrgnt-cmplxty/zero-shot-replication","url":"https://github.com/emrgnt-cmplxty/zero-shot-replication"},{"title":"ethz-privsec/superhuman-ai-consistency","url":"https://github.com/ethz-privsec/superhuman-ai-consistency"},{"title":"ethz-spylab/superhuman-ai-consistency","url":"https://github.com/ethz-spylab/superhuman-ai-consistency"},{"title":"eternityyw/tram-benchmark","url":"https://github.com/eternityyw/tram-benchmark"},{"title":"AUCOHL/RTL-Repo","url":"https://github.com/AUCOHL/RTL-Repo"},{"title":"zach-zhiling-zheng/reticular_chemist","url":"https://github.com/zach-zhiling-zheng/reticular_chemist"},{"title":"lflage/openfactscore","url":"https://github.com/lflage/openfactscore"}],"syntology":{"n":5,"n_ran":2,"n_unverified":3,"n_pointer_only":1}}},{"leaderboard":"/sota/common-sense-reasoning-on-arc-easy","slug":"common-sense-reasoning-on-arc-easy","dataset":"ARC (Easy)","dataset_url":"/dataset/arc","rows_in_archive":47,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"ST-MoE-32B 269B (fine-tuned)","paper_title":"ST-MoE: Designing Stable and Transferable Sparse Expert Models","paper_url":"/paper/designing-effective-sparse-expert-models","paper_date":"2022-02-17","arxiv_id":"2202.08906","code_links":[{"title":"tensorflow/mesh","url":"https://github.com/tensorflow/mesh"},{"title":"xuefuzhao/openmoe","url":"https://github.com/xuefuzhao/openmoe"},{"title":"yikangshen/megablocks","url":"https://github.com/yikangshen/megablocks"}],"syntology":{"n":5,"n_ran":5,"n_unverified":0,"n_pointer_only":5}}},{"leaderboard":"/sota/common-sense-reasoning-on-record","slug":"common-sense-reasoning-on-record","dataset":"ReCoRD","dataset_url":"/dataset/record","rows_in_archive":45,"metrics":["EM","F1"],"first_row_in_archive_order":{"model":"Turing NLR v5 XXL 5.4B (fine-tuned)","paper_title":"Toward Efficient Language Model Pretraining and Downstream Adaptation via Self-Evolution: A Case Study on SuperGLUE","paper_url":"/paper/toward-efficient-language-model-pretraining","paper_date":"2022-12-04","arxiv_id":"2212.01853","code_links":[],"syntology":null}},{"leaderboard":"/sota/common-sense-reasoning-on-commonsenseqa","slug":"common-sense-reasoning-on-commonsenseqa","dataset":"CommonsenseQA","dataset_url":"/dataset/commonsenseqa","rows_in_archive":38,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"GPT-4o (HPT)","paper_title":"Hierarchical Prompting Taxonomy: A Universal Evaluation Framework for Large Language Models Aligned with Human Cognitive Principles","paper_url":"/paper/hierarchical-prompting-taxonomy-a-universal","paper_date":"2024-06-18","arxiv_id":"2406.12644","code_links":[{"title":"devichand579/HPT","url":"https://github.com/devichand579/HPT"}],"syntology":null}},{"leaderboard":"/sota/common-sense-reasoning-on-parus","slug":"common-sense-reasoning-on-parus","dataset":"PARus","dataset_url":"/dataset/parus","rows_in_archive":22,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Human Benchmark","paper_title":"RussianSuperGLUE: A Russian Language Understanding Evaluation Benchmark","paper_url":"/paper/russiansuperglue-a-russian-language","paper_date":"2020-10-29","arxiv_id":"2010.15925","code_links":[{"title":"RussianNLP/RussianSuperGLUE","url":"https://github.com/RussianNLP/RussianSuperGLUE"},{"title":"RussianNLP/MOROCCO","url":"https://github.com/RussianNLP/MOROCCO"}],"syntology":{"n":1,"n_ran":1,"n_unverified":0,"n_pointer_only":0}}},{"leaderboard":"/sota/common-sense-reasoning-on-rucos","slug":"common-sense-reasoning-on-rucos","dataset":"RuCoS","dataset_url":"/dataset/rucos","rows_in_archive":22,"metrics":["Average F1","EM "],"first_row_in_archive_order":{"model":"Human Benchmark","paper_title":"RussianSuperGLUE: A Russian Language Understanding Evaluation Benchmark","paper_url":"/paper/russiansuperglue-a-russian-language","paper_date":"2020-10-29","arxiv_id":"2010.15925","code_links":[{"title":"RussianNLP/RussianSuperGLUE","url":"https://github.com/RussianNLP/RussianSuperGLUE"},{"title":"RussianNLP/MOROCCO","url":"https://github.com/RussianNLP/MOROCCO"}],"syntology":{"n":1,"n_ran":1,"n_unverified":0,"n_pointer_only":0}}},{"leaderboard":"/sota/common-sense-reasoning-on-rwsd","slug":"common-sense-reasoning-on-rwsd","dataset":"RWSD","dataset_url":"/dataset/rwsd","rows_in_archive":22,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Golden Transformer","paper_title":null,"paper_url":null,"paper_date":"","arxiv_id":null,"code_links":[],"syntology":null}},{"leaderboard":"/sota/common-sense-reasoning-on-big-bench","slug":"common-sense-reasoning-on-big-bench","dataset":"BIG-bench (Disambiguation QA)","dataset_url":"/dataset/big-bench","rows_in_archive":9,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"PaLM 2 (few-shot, k=3, Direct)","paper_title":"PaLM 2 Technical Report","paper_url":"/paper/palm-2-technical-report-1","paper_date":"2023-05-17","arxiv_id":"2305.10403","code_links":[{"title":"eternityyw/tram-benchmark","url":"https://github.com/eternityyw/tram-benchmark"}],"syntology":null}},{"leaderboard":"/sota/common-sense-reasoning-on-big-bench-causal","slug":"common-sense-reasoning-on-big-bench-causal","dataset":"BIG-bench (Causal Judgment)","dataset_url":"/dataset/big-bench","rows_in_archive":9,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"PaLM 2 (few-shot, k=3, Direct)","paper_title":"PaLM 2 Technical Report","paper_url":"/paper/palm-2-technical-report-1","paper_date":"2023-05-17","arxiv_id":"2305.10403","code_links":[{"title":"eternityyw/tram-benchmark","url":"https://github.com/eternityyw/tram-benchmark"}],"syntology":null}},{"leaderboard":"/sota/common-sense-reasoning-on-big-bench-date","slug":"common-sense-reasoning-on-big-bench-date","dataset":"BIG-bench (Date Understanding)","dataset_url":"/dataset/big-bench","rows_in_archive":9,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"PaLM 2 (few-shot, k=3, CoT)","paper_title":"PaLM 2 Technical Report","paper_url":"/paper/palm-2-technical-report-1","paper_date":"2023-05-17","arxiv_id":"2305.10403","code_links":[{"title":"eternityyw/tram-benchmark","url":"https://github.com/eternityyw/tram-benchmark"}],"syntology":null}},{"leaderboard":"/sota/common-sense-reasoning-on-big-bench-sports","slug":"common-sense-reasoning-on-big-bench-sports","dataset":"BIG-bench (Sports Understanding)","dataset_url":"/dataset/big-bench","rows_in_archive":8,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"PaLM 2(few-shot, k=3, CoT)","paper_title":"PaLM 2 Technical Report","paper_url":"/paper/palm-2-technical-report-1","paper_date":"2023-05-17","arxiv_id":"2305.10403","code_links":[{"title":"eternityyw/tram-benchmark","url":"https://github.com/eternityyw/tram-benchmark"}],"syntology":null}},{"leaderboard":"/sota/common-sense-reasoning-on-event2mind-test","slug":"common-sense-reasoning-on-event2mind-test","dataset":"Event2Mind test","dataset_url":"/dataset/event2mind","rows_in_archive":7,"metrics":["Average Cross-Ent","BLEU"],"first_row_in_archive_order":{"model":"ConvNet","paper_title":"Event2Mind: Commonsense Inference on Events, Intents, and Reactions","paper_url":"/paper/event2mind-commonsense-inference-on-events","paper_date":"2018-05-17","arxiv_id":"1805.06939","code_links":[],"syntology":null}},{"leaderboard":"/sota/common-sense-reasoning-on-russian-event2mind","slug":"common-sense-reasoning-on-russian-event2mind","dataset":"Russian Event2Mind","dataset_url":"/dataset/russian-event2mind","rows_in_archive":7,"metrics":["recall@10"],"first_row_in_archive_order":{"model":"araneum word2vec (skipgram) + GRU","paper_title":"Event2Mind for Russian: Understanding Emotions and Intents in Texts. Corpus and Model for Evaluation","paper_url":"/paper/event2mind-for-russian-understanding-emotions","paper_date":"2020-06-17","arxiv_id":null,"code_links":[{"title":"Alenush/russian_event2mind","url":"https://github.com/Alenush/russian_event2mind"}],"syntology":null}},{"leaderboard":"/sota/common-sense-reasoning-on-swag","slug":"common-sense-reasoning-on-swag","dataset":"SWAG","dataset_url":"/dataset/swag","rows_in_archive":5,"metrics":["Test","Dev"],"first_row_in_archive_order":{"model":"DeBERTalarge","paper_title":"DeBERTa: Decoding-enhanced BERT with Disentangled Attention","paper_url":"/paper/deberta-decoding-enhanced-bert-with","paper_date":"2020-06-05","arxiv_id":"2006.03654","code_links":[{"title":"huggingface/transformers","url":"https://github.com/huggingface/transformers"},{"title":"microsoft/DeBERTa","url":"https://github.com/microsoft/DeBERTa"},{"title":"osu-nlp-group/mind2web","url":"https://github.com/osu-nlp-group/mind2web"},{"title":"neuralmind-ai/coliee","url":"https://github.com/neuralmind-ai/coliee"},{"title":"huberemanuel/DeBERTa","url":"https://github.com/huberemanuel/DeBERTa"},{"title":"Jason-J-Choi/DeBERTa_TxtClassifier","url":"https://github.com/Jason-J-Choi/DeBERTa_TxtClassifier"},{"title":"2024-MindSpore-1/Code2","url":"https://github.com/2024-MindSpore-1/Code2/tree/main/model-1/deberta"},{"title":"MindCode-4/code-3","url":"https://github.com/MindCode-4/code-3/tree/main/deberta"},{"title":"2024-MindSpore-1/Code2","url":"https://github.com/2024-MindSpore-1/Code2/tree/main/model-1/deberta_v2"},{"title":"MindCode-4/code-3","url":"https://github.com/MindCode-4/code-3/tree/main/deberta_v2"},{"title":"JONGWE1/Paddle-DeBERTa","url":"https://github.com/JONGWE1/Paddle-DeBERTa"},{"title":"Mind23-2/MindCode-161","url":"https://github.com/Mind23-2/MindCode-161"},{"title":"pwc-1/Paper-8","url":"https://github.com/pwc-1/Paper-8/tree/main/deberta"},{"title":"pwc-1/Paper-8","url":"https://github.com/pwc-1/Paper-8/tree/main/deberta_v2"}],"syntology":{"n":13,"n_ran":4,"n_unverified":9,"n_pointer_only":3}}},{"leaderboard":"/sota/common-sense-reasoning-on-big-bench-winowhy","slug":"common-sense-reasoning-on-big-bench-winowhy","dataset":"BIG-bench (Winowhy)","dataset_url":"/dataset/big-bench","rows_in_archive":4,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"PaLM-540B (few-shot, k=5)","paper_title":"PaLM: Scaling Language Modeling with Pathways","paper_url":"/paper/palm-scaling-language-modeling-with-pathways-1","paper_date":"2022-04-05","arxiv_id":"2204.02311","code_links":[{"title":"lucidrains/CoCa-pytorch","url":"https://github.com/lucidrains/CoCa-pytorch"},{"title":"lucidrains/PaLM-pytorch","url":"https://github.com/lucidrains/PaLM-pytorch"},{"title":"google/paxml","url":"https://github.com/google/paxml"},{"title":"foundation-model-stack/fms-fsdp","url":"https://github.com/foundation-model-stack/fms-fsdp"},{"title":"lucidrains/PaLM-jax","url":"https://github.com/lucidrains/PaLM-jax"},{"title":"chrisociepa/allamo","url":"https://github.com/chrisociepa/allamo"},{"title":"conceptofmind/PaLM-flax","url":"https://github.com/conceptofmind/PaLM-flax"}],"syntology":{"n":37,"n_ran":30,"n_unverified":7,"n_pointer_only":0}}},{"leaderboard":"/sota/common-sense-reasoning-on-big-bench-known","slug":"common-sense-reasoning-on-big-bench-known","dataset":"BIG-bench (Known Unknowns)","dataset_url":"/dataset/big-bench","rows_in_archive":3,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"PaLM-540B (few-shot, k=5)","paper_title":"PaLM: Scaling Language Modeling with Pathways","paper_url":"/paper/palm-scaling-language-modeling-with-pathways-1","paper_date":"2022-04-05","arxiv_id":"2204.02311","code_links":[{"title":"lucidrains/CoCa-pytorch","url":"https://github.com/lucidrains/CoCa-pytorch"},{"title":"lucidrains/PaLM-pytorch","url":"https://github.com/lucidrains/PaLM-pytorch"},{"title":"google/paxml","url":"https://github.com/google/paxml"},{"title":"foundation-model-stack/fms-fsdp","url":"https://github.com/foundation-model-stack/fms-fsdp"},{"title":"lucidrains/PaLM-jax","url":"https://github.com/lucidrains/PaLM-jax"},{"title":"chrisociepa/allamo","url":"https://github.com/chrisociepa/allamo"},{"title":"conceptofmind/PaLM-flax","url":"https://github.com/conceptofmind/PaLM-flax"}],"syntology":{"n":37,"n_ran":30,"n_unverified":7,"n_pointer_only":0}}},{"leaderboard":"/sota/common-sense-reasoning-on-big-bench-logical","slug":"common-sense-reasoning-on-big-bench-logical","dataset":"BIG-bench (Logical Sequence)","dataset_url":"/dataset/big-bench","rows_in_archive":2,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla-70B (few-shot, k=5)","paper_title":"Training Compute-Optimal Large Language Models","paper_url":"/paper/training-compute-optimal-large-language","paper_date":"2022-03-29","arxiv_id":"2203.15556","code_links":[{"title":"karpathy/llama2.c","url":"https://github.com/karpathy/llama2.c"},{"title":"nkluge-correa/teenytinyllama","url":"https://github.com/nkluge-correa/teenytinyllama"}],"syntology":{"n":11,"n_ran":8,"n_unverified":3,"n_pointer_only":4}}},{"leaderboard":"/sota/common-sense-reasoning-on-event2mind-dev","slug":"common-sense-reasoning-on-event2mind-dev","dataset":"Event2Mind dev","dataset_url":"/dataset/event2mind","rows_in_archive":2,"metrics":["Average Cross-Ent"],"first_row_in_archive_order":{"model":"ConvNet","paper_title":"Event2Mind: Commonsense Inference on Events, Intents, and Reactions","paper_url":"/paper/event2mind-commonsense-inference-on-events","paper_date":"2018-05-17","arxiv_id":"1805.06939","code_links":[],"syntology":null}},{"leaderboard":"/sota/common-sense-reasoning-on-codah","slug":"common-sense-reasoning-on-codah","dataset":"CODAH","dataset_url":"/dataset/codah","rows_in_archive":1,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"BERT Large","paper_title":"CODAH: An Adversarially Authored Question-Answer Dataset for Common Sense","paper_url":"/paper/aqua-an-adversarially-authored-question","paper_date":"2019-04-08","arxiv_id":"1904.04365","code_links":[{"title":"Websail-NU/AQuA","url":"https://github.com/Websail-NU/AQuA"},{"title":"iit-nlp-research/chatgpt-crawler","url":"https://github.com/iit-nlp-research/chatgpt-crawler"}],"syntology":null}},{"leaderboard":"/sota/common-sense-reasoning-on-crowdsource-qa-1","slug":"common-sense-reasoning-on-crowdsource-qa-1","dataset":"CrowdSource QA","dataset_url":null,"rows_in_archive":1,"metrics":["MSE"],"first_row_in_archive_order":{"model":"BERT","paper_title":"Predicting Subjective Features of Questions of QA Websites using BERT","paper_url":"/paper/predicting-subjective-features-from-questions","paper_date":"2020-02-24","arxiv_id":"2002.10107","code_links":[{"title":"huggingface/transformers","url":"https://github.com/huggingface/transformers"},{"title":"tensorflow/models","url":"https://github.com/tensorflow/models/tree/master/official/nlp"},{"title":"google-research/bert","url":"https://github.com/google-research/bert"},{"title":"graykode/nlp-tutorial","url":"https://github.com/graykode/nlp-tutorial"},{"title":"Moradnejad/Predicting-Subjective-Features-on-QA-Websites","url":"https://github.com/Moradnejad/Predicting-Subjective-Features-on-QA-Websites"}],"syntology":null}},{"leaderboard":"/sota/common-sense-reasoning-on-visual-dialog-v0-9","slug":"common-sense-reasoning-on-visual-dialog-v0-9","dataset":"Visual Dialog v0.9","dataset_url":"/dataset/visdial","rows_in_archive":1,"metrics":["1 in 10 R@5"],"first_row_in_archive_order":{"model":"NMN [kottur2018visual]","paper_title":"Visual Coreference Resolution in Visual Dialog using Neural Module Networks","paper_url":"/paper/visual-coreference-resolution-in-visual","paper_date":"2018-09-06","arxiv_id":"1809.01816","code_links":[{"title":"facebookresearch/corefnmn","url":"https://github.com/facebookresearch/corefnmn"}],"syntology":null}},{"leaderboard":"/sota/common-sense-reasoning-on-visual-dialog-v09","slug":"common-sense-reasoning-on-visual-dialog-v09","dataset":"Visual Dialog  v0.9","dataset_url":"/dataset/visdial","rows_in_archive":1,"metrics":["1 in 10 R@5","Recall@10"],"first_row_in_archive_order":{"model":"PDUN","paper_title":"Probabilistic framework for solving Visual Dialog","paper_url":"/paper/probabilistic-framework-for-solving-visual","paper_date":"2019-09-11","arxiv_id":"1909.04800","code_links":[],"syntology":null}},{"leaderboard":"/sota/common-sense-reasoning-on-winogavil","slug":"common-sense-reasoning-on-winogavil","dataset":"WinoGAViL","dataset_url":"/dataset/winogavil","rows_in_archive":1,"metrics":["Jaccard Index"],"first_row_in_archive_order":{"model":"ViLT","paper_title":"WinoGAViL: Gamified Association Benchmark to Challenge Vision-and-Language Models","paper_url":"/paper/winogavil-gamified-association-benchmark-to","paper_date":"2022-07-25","arxiv_id":"2207.12576","code_links":[{"title":"winogavil/winogavil-experiments","url":"https://github.com/winogavil/winogavil-experiments"}],"syntology":{"n":2,"n_ran":1,"n_unverified":1,"n_pointer_only":0}}}],"datasets":[{"url":"/dataset/winogrande","name":"WinoGrande","full_name":"","num_papers_in_archive":703},{"url":"/dataset/commonsenseqa","name":"CommonsenseQA","full_name":"CSQA","num_papers_in_archive":483},{"url":"/dataset/big-bench","name":"BIG-bench","full_name":"Beyond the Imitation Game Benchmark","num_papers_in_archive":349},{"url":"/dataset/arc","name":"ARC (AI2 Reasoning Challenge)","full_name":"","num_papers_in_archive":178},{"url":"/dataset/swag","name":"SWAG","full_name":"Situations With Adversarial Generations","num_papers_in_archive":163},{"url":"/dataset/visdial","name":"VisDial","full_name":"Visual Dialog","num_papers_in_archive":159},{"url":"/dataset/record","name":"ReCoRD","full_name":"ReCoRD","num_papers_in_archive":111},{"url":"/dataset/cos-e","name":"CoS-E","full_name":"Commonsense Explanations Dataset","num_papers_in_archive":46},{"url":"/dataset/iconqa","name":"IconQA","full_name":"Icon Question Answering","num_papers_in_archive":42},{"url":"/dataset/phyre","name":"PHYRE","full_name":"PHYsical REasoning","num_papers_in_archive":35},{"url":"/dataset/creak","name":"CREAK","full_name":"","num_papers_in_archive":30},{"url":"/dataset/codah","name":"CODAH","full_name":"COmmonsense Dataset Adversarially-authored by Humans","num_papers_in_archive":29},{"url":"/dataset/onestopenglish","name":"OneStopEnglish","full_name":"","num_papers_in_archive":29},{"url":"/dataset/xstorycloze","name":"XStoryCloze","full_name":"","num_papers_in_archive":26},{"url":"/dataset/mcscript","name":"MCScript","full_name":"MCScript","num_papers_in_archive":24},{"url":"/dataset/moral-stories","name":"Moral Stories","full_name":"","num_papers_in_archive":24},{"url":"/dataset/recipeqa","name":"RecipeQA","full_name":"","num_papers_in_archive":24},{"url":"/dataset/riddle-sense","name":"RiddleSense","full_name":"","num_papers_in_archive":23},{"url":"/dataset/event2mind","name":"Event2Mind","full_name":"","num_papers_in_archive":21},{"url":"/dataset/x-csqa","name":"X-CSQA","full_name":"","num_papers_in_archive":18},{"url":"/dataset/stepgame","name":"StepGame","full_name":"StepGame","num_papers_in_archive":15},{"url":"/dataset/whoops","name":"WHOOPS!","full_name":"","num_papers_in_archive":14},{"url":"/dataset/com2sense","name":"Com2Sense","full_name":"Complementary Commonsense","num_papers_in_archive":11},{"url":"/dataset/housekeep","name":"Housekeep","full_name":"","num_papers_in_archive":11},{"url":"/dataset/protoqa","name":"ProtoQA","full_name":null,"num_papers_in_archive":11},{"url":"/dataset/timedial","name":"TimeDial","full_name":"","num_papers_in_archive":11},{"url":"/dataset/cc-stories","name":"CC-Stories","full_name":"CC-Stories","num_papers_in_archive":10},{"url":"/dataset/criticbench","name":"CriticBench","full_name":"","num_papers_in_archive":10},{"url":"/dataset/fig-qa","name":"Fig-QA","full_name":"","num_papers_in_archive":10},{"url":"/dataset/rainbow","name":"Rainbow","full_name":"","num_papers_in_archive":10},{"url":"/dataset/parus","name":"PARus","full_name":"Choice of Plausible Alternatives for Russian language","num_papers_in_archive":7},{"url":"/dataset/rwsd","name":"RWSD","full_name":"The Winograd Schema Challenge (Russian)","num_papers_in_archive":7},{"url":"/dataset/cite","name":"CITE","full_name":"","num_papers_in_archive":6},{"url":"/dataset/omics","name":"OMICS","full_name":"Open Mind Indoor Common Sense","num_papers_in_archive":6},{"url":"/dataset/smart-101","name":"SMART-101","full_name":"Simple Multimodal Algorithmic Reasoning Task Dataset","num_papers_in_archive":6},{"url":"/dataset/numersense","name":"NumerSense","full_name":"","num_papers_in_archive":5},{"url":"/dataset/rucos","name":"RuCoS","full_name":"Russian Reading Comprehension with Commonsense Reasoning","num_papers_in_archive":5},{"url":"/dataset/sarcasm-corpus-v2","name":"Sarcasm Corpus V2","full_name":"","num_papers_in_archive":5},{"url":"/dataset/winogavil","name":"WinoGAViL","full_name":"","num_papers_in_archive":5},{"url":"/dataset/mindgames","name":"Mindgames","full_name":"","num_papers_in_archive":4},{"url":"/dataset/bcopa-ce","name":"BCOPA-CE","full_name":"A Balanced COPA Test Set with cause-effect as alternatives","num_papers_in_archive":3},{"url":"/dataset/pacs-commonsense","name":"Physical Audiovisual CommonSense","full_name":"","num_papers_in_archive":3},{"url":"/dataset/chegeka","name":"CheGeKa","full_name":"CheGeKa","num_papers_in_archive":2},{"url":"/dataset/cofar","name":"COFAR","full_name":"Commonsense and Factual Reasoning in Image Search","num_papers_in_archive":2},{"url":"/dataset/g-vue","name":"G-VUE","full_name":"General-purpose Visual Understanding Evaluation","num_papers_in_archive":2},{"url":"/dataset/machine-number-sense","name":"Machine Number Sense","full_name":"","num_papers_in_archive":2},{"url":"/dataset/qure","name":"QuRe","full_name":"","num_papers_in_archive":2},{"url":"/dataset/situation-puzzle","name":"Situation Puzzle","full_name":"","num_papers_in_archive":2},{"url":"/dataset/accord-csqa-0-5","name":"ACCORD CSQA 0-5","full_name":"","num_papers_in_archive":1},{"url":"/dataset/advice-seeking-questions","name":"Advice Seeking Questions","full_name":null,"num_papers_in_archive":1},{"url":"/dataset/discosense","name":"DiscoSense","full_name":"","num_papers_in_archive":1},{"url":"/dataset/dpgmedia2019","name":"DpgMedia2019","full_name":null,"num_papers_in_archive":1},{"url":"/dataset/russian-event2mind","name":"Russian Event2Mind","full_name":"","num_papers_in_archive":1},{"url":"/dataset/this-is-not-a-dataset","name":"This is not a Dataset","full_name":"This is not a Dataset: A Large Negation Benchmark to Challenge Large Language Models","num_papers_in_archive":1},{"url":"/dataset/winograd-automatic","name":"Winograd Automatic","full_name":"Winograd","num_papers_in_archive":1},{"url":"/dataset/cvr-tasks","name":"Compositional Visual Reasoning (CVR)","full_name":"","num_papers_in_archive":0}],"subtasks":[{"url":"/task/anachronisms","name":"Anachronisms"},{"url":"/task/causal-judgment","name":"Causal Judgment"},{"url":"/task/crash-blossom","name":"Crash Blossom"},{"url":"/task/crass-ai","name":"Crass AI"},{"url":"/task/disambiguation-qa","name":"Disambiguation QA"},{"url":"/task/discourse-marker-prediction","name":"Discourse Marker Prediction"},{"url":"/task/empirical-judgments","name":"Empirical Judgments"},{"url":"/task/hellaswag","name":"HellaSwag"},{"url":"/task/irony-identification","name":"Irony Identification"},{"url":"/task/multiview-contextual-commonsense-inference","name":"Multiview Contextual Commonsense Inference"},{"url":"/task/physical-commonsense-reasoning","name":"Physical Commonsense Reasoning"},{"url":"/task/riddle-sense","name":"Riddle Sense"},{"url":"/task/timedial","name":"Timedial"},{"url":"/task/understanding-fables","name":"Understanding Fables"},{"url":"/task/visual-commonsense-tests","name":"Visual Commonsense Tests"},{"url":"/task/winogrande","name":"Winogrande"},{"url":"/task/winowhy","name":"Winowhy"}],"parent_tasks":[],"papers":{"order":"repositories listed in the archive (desc), then date (desc); the archive holds no stars","population":"papers tagged with this task that list at least one repository in the archive","shown":30,"of":325,"tagged_in_all":939,"items":[{"url":"/paper/bert-pre-training-of-deep-bidirectional","title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding","date":"2018-10-11","arxiv_id":"1810.04805","repositories_listed":534,"syntology":{"n":659,"n_ran":204,"n_unverified":455,"n_pointer_only":149}},{"url":"/paper/language-models-are-few-shot-learners","title":"Language Models are Few-Shot Learners","date":"2020-05-28","arxiv_id":"2005.14165","repositories_listed":67,"syntology":{"n":65,"n_ran":15,"n_unverified":50,"n_pointer_only":4}},{"url":"/paper/roberta-a-robustly-optimized-bert-pretraining","title":"RoBERTa: A Robustly Optimized BERT Pretraining Approach","date":"2019-07-26","arxiv_id":"1907.11692","repositories_listed":67,"syntology":{"n":48,"n_ran":22,"n_unverified":26,"n_pointer_only":23}},{"url":"/paper/llama-open-and-efficient-foundation-language-1","title":"LLaMA: Open and Efficient Foundation Language Models","date":"2023-02-27","arxiv_id":"2302.13971","repositories_listed":57,"syntology":{"n":58,"n_ran":26,"n_unverified":32,"n_pointer_only":4}},{"url":"/paper/exploring-the-limits-of-transfer-learning","title":"Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer","date":"2019-10-23","arxiv_id":"1910.10683","repositories_listed":57,"syntology":{"n":31,"n_ran":2,"n_unverified":29,"n_pointer_only":0}},{"url":"/paper/albert-a-lite-bert-for-self-supervised","title":"ALBERT: A Lite BERT for Self-supervised Learning of Language Representations","date":"2019-09-26","arxiv_id":"1909.11942","repositories_listed":48,"syntology":{"n":126,"n_ran":46,"n_unverified":80,"n_pointer_only":22}},{"url":"/paper/mamba-linear-time-sequence-modeling-with","title":"Mamba: Linear-Time Sequence Modeling with Selective State Spaces","date":"2023-12-01","arxiv_id":"2312.00752","repositories_listed":35,"syntology":{"n":62,"n_ran":18,"n_unverified":44,"n_pointer_only":28}},{"url":"/paper/language-models-are-unsupervised-multitask","title":"Language Models are Unsupervised Multitask Learners","date":"2019-02-14","arxiv_id":null,"repositories_listed":21,"syntology":null},{"url":"/paper/chain-of-thought-prompting-elicits-reasoning","title":"Chain-of-Thought Prompting Elicits Reasoning in Large Language Models","date":"2022-01-28","arxiv_id":"2201.11903","repositories_listed":19,"syntology":{"n":7,"n_ran":2,"n_unverified":5,"n_pointer_only":0}},{"url":"/paper/a-neural-conversational-model","title":"A Neural Conversational Model","date":"2015-06-19","arxiv_id":"1506.05869","repositories_listed":18,"syntology":{"n":4,"n_ran":2,"n_unverified":2,"n_pointer_only":3}},{"url":"/paper/deberta-decoding-enhanced-bert-with","title":"DeBERTa: Decoding-enhanced BERT with Disentangled Attention","date":"2020-06-05","arxiv_id":"2006.03654","repositories_listed":14,"syntology":{"n":13,"n_ran":4,"n_unverified":9,"n_pointer_only":3}},{"url":"/paper/awq-activation-aware-weight-quantization-for","title":"AWQ: Activation-aware Weight Quantization for LLM Compression and Acceleration","date":"2023-06-01","arxiv_id":"2306.00978","repositories_listed":12,"syntology":{"n":18,"n_ran":12,"n_unverified":6,"n_pointer_only":2}},{"url":"/paper/gpt-4-technical-report-1","title":"GPT-4 Technical Report","date":"2023-03-15","arxiv_id":"2303.08774","repositories_listed":11,"syntology":{"n":5,"n_ran":2,"n_unverified":3,"n_pointer_only":1}},{"url":"/paper/winogrande-an-adversarial-winograd-schema","title":"WinoGrande: An Adversarial Winograd Schema Challenge at Scale","date":"2019-07-24","arxiv_id":"1907.10641","repositories_listed":10,"syntology":null},{"url":"/paper/luke-deep-contextualized-entity","title":"LUKE: Deep Contextualized Entity Representations with Entity-aware Self-attention","date":"2020-10-02","arxiv_id":"2010.01057","repositories_listed":9,"syntology":{"n":10,"n_ran":3,"n_unverified":7,"n_pointer_only":0}},{"url":"/paper/finetuned-language-models-are-zero-shot","title":"Finetuned Language Models Are Zero-Shot Learners","date":"2021-09-03","arxiv_id":"2109.01652","repositories_listed":8,"syntology":{"n":1,"n_ran":0,"n_unverified":1,"n_pointer_only":0}},{"url":"/paper/mt5-a-massively-multilingual-pre-trained-text","title":"mT5: A massively multilingual pre-trained text-to-text transformer","date":"2020-10-22","arxiv_id":"2010.11934","repositories_listed":8,"syntology":{"n":13,"n_ran":0,"n_unverified":13,"n_pointer_only":0}},{"url":"/paper/palm-scaling-language-modeling-with-pathways-1","title":"PaLM: Scaling Language Modeling with Pathways","date":"2022-04-05","arxiv_id":"2204.02311","repositories_listed":7,"syntology":{"n":37,"n_ran":30,"n_unverified":7,"n_pointer_only":0}},{"url":"/paper/qwen2-5-technical-report","title":"Qwen2.5 Technical Report","date":"2024-12-19","arxiv_id":"2412.15115","repositories_listed":6,"syntology":{"n":3,"n_ran":1,"n_unverified":2,"n_pointer_only":0}},{"url":"/paper/mixtral-of-experts","title":"Mixtral of Experts","date":"2024-01-08","arxiv_id":"2401.04088","repositories_listed":6,"syntology":{"n":5,"n_ran":5,"n_unverified":0,"n_pointer_only":0}},{"url":"/paper/mistral-7b","title":"Mistral 7B","date":"2023-10-10","arxiv_id":"2310.06825","repositories_listed":6,"syntology":{"n":11,"n_ran":9,"n_unverified":2,"n_pointer_only":1}},{"url":"/paper/massive-language-models-can-be-accurately","title":"SparseGPT: Massive Language Models Can Be Accurately Pruned in One-Shot","date":"2023-01-02","arxiv_id":"2301.00774","repositories_listed":6,"syntology":{"n":12,"n_ran":2,"n_unverified":10,"n_pointer_only":9}},{"url":"/paper/beyond-the-imitation-game-quantifying-and","title":"Beyond the Imitation Game: Quantifying and extrapolating the capabilities of language models","date":"2022-06-09","arxiv_id":"2206.04615","repositories_listed":6,"syntology":{"n":3,"n_ran":0,"n_unverified":3,"n_pointer_only":0}},{"url":"/paper/qa-gnn-reasoning-with-language-models-and","title":"QA-GNN: Reasoning with Language Models and Knowledge Graphs for Question Answering","date":"2021-04-13","arxiv_id":"2104.06378","repositories_listed":6,"syntology":{"n":25,"n_ran":1,"n_unverified":24,"n_pointer_only":4}},{"url":"/paper/temporal-relational-reasoning-in-videos","title":"Temporal Relational Reasoning in Videos","date":"2017-11-22","arxiv_id":"1711.08496","repositories_listed":5,"syntology":{"n":3,"n_ran":2,"n_unverified":1,"n_pointer_only":3}},{"url":"/paper/the-something-something-video-database-for","title":"The \"something something\" video database for learning and evaluating visual common sense","date":"2017-06-13","arxiv_id":"1706.04261","repositories_listed":5,"syntology":null},{"url":"/paper/gated-delta-networks-improving-mamba2-with","title":"Gated Delta Networks: Improving Mamba2 with Delta Rule","date":"2024-12-09","arxiv_id":"2412.06464","repositories_listed":4,"syntology":{"n":13,"n_ran":5,"n_unverified":8,"n_pointer_only":7}},{"url":"/paper/smart-agent-based-modeling-on-the-use-of","title":"Smart Agent-Based Modeling: On the Use of Large Language Models in Computer Simulations","date":"2023-11-10","arxiv_id":"2311.06330","repositories_listed":4,"syntology":{"n":2,"n_ran":2,"n_unverified":0,"n_pointer_only":2}},{"url":"/paper/pythia-a-suite-for-analyzing-large-language","title":"Pythia: A Suite for Analyzing Large Language Models Across Training and Scaling","date":"2023-04-03","arxiv_id":"2304.01373","repositories_listed":4,"syntology":null},{"url":"/paper/large-language-models-are-zero-shot-reasoners","title":"Large Language Models are Zero-Shot Reasoners","date":"2022-05-24","arxiv_id":"2205.11916","repositories_listed":4,"syntology":{"n":4,"n_ran":0,"n_unverified":4,"n_pointer_only":1}}],"syntology_records":26,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-09-24T18:15:14+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"}},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"}}