{"url":"/task/logical-reasoning","name":"Logical Reasoning","slug":"logical-reasoning","description_markdown":null,"categories":[{"name":"Methodology","url":"/area/methodology"},{"name":"Miscellaneous","url":"/area/miscellaneous"},{"name":"Reasoning","url":"/area/reasoning"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":747,"papers_with_code":330,"benchmarks":10,"benchmark_tables_in_archive":10,"benchmark_tables_shown":10,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":18,"subtasks":22,"parent_tasks":0},"benchmarks":[{"leaderboard":"/sota/logical-reasoning-on-lingoly","slug":"logical-reasoning-on-lingoly","dataset":"LingOly","dataset_url":"/dataset/lingoly","rows_in_archive":11,"metrics":["Delta_NoContext","Exact Match Accuracy"],"first_row_in_archive_order":{"model":"Claude Opus","paper_title":"LINGOLY: A Benchmark of Olympiad-Level Linguistic Reasoning Puzzles in Low-Resource and Extinct Languages","paper_url":"/paper/lingoly-a-benchmark-of-olympiad-level","paper_date":"2024-06-10","arxiv_id":"2406.06196","code_links":[{"title":"am-bean/lingOly","url":"https://github.com/am-bean/lingOly"}],"syntology":{"n":2,"n_ran":2,"n_unverified":0,"n_pointer_only":2}}},{"leaderboard":"/sota/logical-reasoning-on-big-bench-formal","slug":"logical-reasoning-on-big-bench-formal","dataset":"BIG-bench (Formal Fallacies Syllogisms Negation)","dataset_url":"/dataset/big-bench","rows_in_archive":9,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"PaLM 2 (few-shot, k=3, Direct)","paper_title":"PaLM 2 Technical Report","paper_url":"/paper/palm-2-technical-report-1","paper_date":"2023-05-17","arxiv_id":"2305.10403","code_links":[{"title":"eternityyw/tram-benchmark","url":"https://github.com/eternityyw/tram-benchmark"}],"syntology":null}},{"leaderboard":"/sota/logical-reasoning-on-big-bench-penguins-in-a","slug":"logical-reasoning-on-big-bench-penguins-in-a","dataset":"BIG-bench (Penguins In A Table)","dataset_url":"/dataset/big-bench","rows_in_archive":9,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"PaLM 2 (few-shot, k=3, CoT)","paper_title":"PaLM 2 Technical Report","paper_url":"/paper/palm-2-technical-report-1","paper_date":"2023-05-17","arxiv_id":"2305.10403","code_links":[{"title":"eternityyw/tram-benchmark","url":"https://github.com/eternityyw/tram-benchmark"}],"syntology":null}},{"leaderboard":"/sota/logical-reasoning-on-big-bench-reasoning","slug":"logical-reasoning-on-big-bench-reasoning","dataset":"BIG-bench (Reasoning About Colored Objects)","dataset_url":"/dataset/big-bench","rows_in_archive":9,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"PaLM 2 (few-shot, k=3, CoT)","paper_title":"PaLM 2 Technical Report","paper_url":"/paper/palm-2-technical-report-1","paper_date":"2023-05-17","arxiv_id":"2305.10403","code_links":[{"title":"eternityyw/tram-benchmark","url":"https://github.com/eternityyw/tram-benchmark"}],"syntology":null}},{"leaderboard":"/sota/logical-reasoning-on-big-bench-temporal","slug":"logical-reasoning-on-big-bench-temporal","dataset":"BIG-bench (Temporal Sequences)","dataset_url":"/dataset/big-bench","rows_in_archive":9,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"PaLM 2 (few-shot, k=3, CoT)","paper_title":"PaLM 2 Technical Report","paper_url":"/paper/palm-2-technical-report-1","paper_date":"2023-05-17","arxiv_id":"2305.10403","code_links":[{"title":"eternityyw/tram-benchmark","url":"https://github.com/eternityyw/tram-benchmark"}],"syntology":null}},{"leaderboard":"/sota/logical-reasoning-on-big-bench-logic-grid","slug":"logical-reasoning-on-big-bench-logic-grid","dataset":"BIG-bench (Logic Grid Puzzle)","dataset_url":"/dataset/big-bench","rows_in_archive":4,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla-70B (few-shot, k=5)","paper_title":"Training Compute-Optimal Large Language Models","paper_url":"/paper/training-compute-optimal-large-language","paper_date":"2022-03-29","arxiv_id":"2203.15556","code_links":[{"title":"karpathy/llama2.c","url":"https://github.com/karpathy/llama2.c"},{"title":"nkluge-correa/teenytinyllama","url":"https://github.com/nkluge-correa/teenytinyllama"}],"syntology":{"n":11,"n_ran":8,"n_unverified":3,"n_pointer_only":4}}},{"leaderboard":"/sota/logical-reasoning-on-big-bench-strategyqa","slug":"logical-reasoning-on-big-bench-strategyqa","dataset":"BIG-bench (StrategyQA)","dataset_url":"/dataset/big-bench","rows_in_archive":4,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"PaLM-540B (few-shot, k=5)","paper_title":"PaLM: Scaling Language Modeling with Pathways","paper_url":"/paper/palm-scaling-language-modeling-with-pathways-1","paper_date":"2022-04-05","arxiv_id":"2204.02311","code_links":[{"title":"lucidrains/CoCa-pytorch","url":"https://github.com/lucidrains/CoCa-pytorch"},{"title":"lucidrains/PaLM-pytorch","url":"https://github.com/lucidrains/PaLM-pytorch"},{"title":"google/paxml","url":"https://github.com/google/paxml"},{"title":"foundation-model-stack/fms-fsdp","url":"https://github.com/foundation-model-stack/fms-fsdp"},{"title":"lucidrains/PaLM-jax","url":"https://github.com/lucidrains/PaLM-jax"},{"title":"chrisociepa/allamo","url":"https://github.com/chrisociepa/allamo"},{"title":"conceptofmind/PaLM-flax","url":"https://github.com/conceptofmind/PaLM-flax"}],"syntology":{"n":37,"n_ran":30,"n_unverified":7,"n_pointer_only":0}}},{"leaderboard":"/sota/logical-reasoning-on-ruworldtree","slug":"logical-reasoning-on-ruworldtree","dataset":"RuWorldTree","dataset_url":"/dataset/ruworldtree","rows_in_archive":4,"metrics":["Accuracy "],"first_row_in_archive_order":{"model":"Human benchmark","paper_title":"TAPE: Assessing Few-shot Russian Language Understanding","paper_url":"/paper/tape-assessing-few-shot-russian-language","paper_date":"2022-10-23","arxiv_id":"2210.12813","code_links":[{"title":"RussianNLP/TAPE","url":"https://github.com/RussianNLP/TAPE"}],"syntology":null}},{"leaderboard":"/sota/logical-reasoning-on-winograd-automatic","slug":"logical-reasoning-on-winograd-automatic","dataset":"Winograd Automatic","dataset_url":"/dataset/winograd-automatic","rows_in_archive":4,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Human benchmark","paper_title":"TAPE: Assessing Few-shot Russian Language Understanding","paper_url":"/paper/tape-assessing-few-shot-russian-language","paper_date":"2022-10-23","arxiv_id":"2210.12813","code_links":[{"title":"RussianNLP/TAPE","url":"https://github.com/RussianNLP/TAPE"}],"syntology":null}},{"leaderboard":"/sota/logical-reasoning-on-big-bench-logical","slug":"logical-reasoning-on-big-bench-logical","dataset":"BIG-bench (Logical Fallacy Detection)","dataset_url":"/dataset/big-bench","rows_in_archive":2,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla-70B (few-shot, k=5)","paper_title":"Training Compute-Optimal Large Language Models","paper_url":"/paper/training-compute-optimal-large-language","paper_date":"2022-03-29","arxiv_id":"2203.15556","code_links":[{"title":"karpathy/llama2.c","url":"https://github.com/karpathy/llama2.c"},{"title":"nkluge-correa/teenytinyllama","url":"https://github.com/nkluge-correa/teenytinyllama"}],"syntology":{"n":11,"n_ran":8,"n_unverified":3,"n_pointer_only":4}}}],"datasets":[{"url":"/dataset/big-bench","name":"BIG-bench","full_name":"Beyond the Imitation Game Benchmark","num_papers_in_archive":349},{"url":"/dataset/gsm-plus","name":"GSM-Plus","full_name":"","num_papers_in_archive":17},{"url":"/dataset/smart-101","name":"SMART-101","full_name":"Simple Multimodal Algorithmic Reasoning Task Dataset","num_papers_in_archive":6},{"url":"/dataset/fld","name":"FLD","full_name":"Formal Logic Deduction","num_papers_in_archive":5},{"url":"/dataset/lingoly","name":"LingOly","full_name":"","num_papers_in_archive":4},{"url":"/dataset/multiq","name":"MultiQ","full_name":"MultiQ","num_papers_in_archive":3},{"url":"/dataset/chegeka","name":"CheGeKa","full_name":"CheGeKa","num_papers_in_archive":2},{"url":"/dataset/ethics-2","name":"Ethics (per ethics)","full_name":"","num_papers_in_archive":2},{"url":"/dataset/qure","name":"QuRe","full_name":"","num_papers_in_archive":2},{"url":"/dataset/ruopenbookqa","name":"RuOpenBookQA","full_name":"RuOpenBookQA","num_papers_in_archive":2},{"url":"/dataset/ruworldtree","name":"RuWorldTree","full_name":"RuWorldTree","num_papers_in_archive":2},{"url":"/dataset/truthquest","name":"TruthQuest","full_name":"","num_papers_in_archive":2},{"url":"/dataset/inferential-strategies","name":"Inferential-Strategies","full_name":"","num_papers_in_archive":1},{"url":"/dataset/justlogic","name":"JustLogic","full_name":"","num_papers_in_archive":1},{"url":"/dataset/oxford-ontology-library","name":"Oxford Ontology Library","full_name":"","num_papers_in_archive":1},{"url":"/dataset/polymath","name":"PolyMATH","full_name":"","num_papers_in_archive":1},{"url":"/dataset/rebus","name":"REBUS","full_name":"A Robust Evaluation Benchmark of Understanding Symbols","num_papers_in_archive":1},{"url":"/dataset/winograd-automatic","name":"Winograd Automatic","full_name":"Winograd","num_papers_in_archive":1}],"subtasks":[{"url":"/task/analytic-entailment","name":"Analytic Entailment"},{"url":"/task/checkmate-in-one","name":"Checkmate In One"},{"url":"/task/code-line-descriptions","name":"Code Line Descriptions"},{"url":"/task/college-mathematics","name":"College Mathematics"},{"url":"/task/date-understanding","name":"Date Understanding"},{"url":"/task/elementary-mathematics","name":"Elementary Mathematics"},{"url":"/task/entailed-polarity","name":"Entailed Polarity"},{"url":"/task/epistemic-reasoning","name":"Epistemic Reasoning"},{"url":"/task/evaluating-information-essentiality","name":"Evaluating Information Essentiality"},{"url":"/task/logic-grid-puzzle","name":"Logic Grid Puzzle"},{"url":"/task/logical-args","name":"Logical Args"},{"url":"/task/logical-fallacy-detection","name":"Logical Fallacy Detection"},{"url":"/task/logical-sequence","name":"Logical Sequence"},{"url":"/task/metaphor-boolean","name":"Metaphor Boolean"},{"url":"/task/navigate","name":"Navigate"},{"url":"/task/novel-concepts","name":"Novel Concepts"},{"url":"/task/penguins-in-a-table","name":"Penguins In A Table"},{"url":"/task/physical-intuition","name":"Physical Intuition"},{"url":"/task/presuppositions-as-nli","name":"Presuppositions As NLI"},{"url":"/task/reasoning-about-colored-objects","name":"Reasoning About Colored Objects"},{"url":"/task/strategyqa","name":"StrategyQA"},{"url":"/task/temporal-sequences","name":"Temporal Sequences"}],"parent_tasks":[],"papers":{"order":"repositories listed in the archive (desc), then date (desc); the archive holds no stars","population":"papers tagged with this task that list at least one repository in the archive","shown":30,"of":330,"tagged_in_all":747,"items":[{"url":"/paper/beta-embeddings-for-multi-hop-logical","title":"Beta Embeddings for Multi-Hop Logical Reasoning in Knowledge Graphs","date":"2020-10-22","arxiv_id":"2010.11465","repositories_listed":9,"syntology":{"n":14,"n_ran":8,"n_unverified":6,"n_pointer_only":1}},{"url":"/paper/palm-scaling-language-modeling-with-pathways-1","title":"PaLM: Scaling Language Modeling with Pathways","date":"2022-04-05","arxiv_id":"2204.02311","repositories_listed":7,"syntology":{"n":37,"n_ran":30,"n_unverified":7,"n_pointer_only":0}},{"url":"/paper/mmmu-a-massive-multi-discipline-multimodal","title":"MMMU: A Massive Multi-discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI","date":"2023-11-27","arxiv_id":"2311.16502","repositories_listed":5,"syntology":{"n":14,"n_ran":6,"n_unverified":8,"n_pointer_only":1}},{"url":"/paper/scaling-synthetic-data-creation-with","title":"Scaling Synthetic Data Creation with 1,000,000,000 Personas","date":"2024-06-28","arxiv_id":"2406.20094","repositories_listed":4,"syntology":{"n":4,"n_ran":1,"n_unverified":3,"n_pointer_only":1}},{"url":"/paper/large-language-models-are-zero-shot-reasoners","title":"Large Language Models are Zero-Shot Reasoners","date":"2022-05-24","arxiv_id":"2205.11916","repositories_listed":4,"syntology":{"n":4,"n_ran":0,"n_unverified":4,"n_pointer_only":1}},{"url":"/paper/hunyuan-large-an-open-source-moe-model-with","title":"Hunyuan-Large: An Open-Source MoE Model with 52 Billion Activated Parameters by Tencent","date":"2024-11-04","arxiv_id":"2411.02265","repositories_listed":3,"syntology":null},{"url":"/paper/knowledge-fusion-of-large-language-models","title":"Knowledge Fusion of Large Language Models","date":"2024-01-19","arxiv_id":"2401.10491","repositories_listed":3,"syntology":{"n":4,"n_ran":1,"n_unverified":3,"n_pointer_only":4}},{"url":"/paper/learning-deductive-reasoning-from-synthetic","title":"Learning Deductive Reasoning from Synthetic Corpus based on Formal Logic","date":"2023-08-11","arxiv_id":"2308.07336","repositories_listed":3,"syntology":{"n":2,"n_ran":0,"n_unverified":2,"n_pointer_only":0}},{"url":"/paper/metalogic-logical-reasoning-explanations-with","title":"MetaLogic: Logical Reasoning Explanations with Fine-Grained Structure","date":"2022-10-22","arxiv_id":"2210.12487","repositories_listed":3,"syntology":null},{"url":"/paper/scaling-language-models-methods-analysis-1","title":"Scaling Language Models: Methods, Analysis & Insights from Training Gopher","date":"2021-12-08","arxiv_id":"2112.11446","repositories_listed":3,"syntology":null},{"url":"/paper/neural-logic-reasoning","title":"Neural Logic Reasoning","date":"2020-08-20","arxiv_id":"2008.09514","repositories_listed":3,"syntology":{"n":3,"n_ran":0,"n_unverified":3,"n_pointer_only":0}},{"url":"/paper/neural-collaborative-reasoning","title":"Neural Collaborative Reasoning","date":"2020-05-16","arxiv_id":"2005.08129","repositories_listed":3,"syntology":null},{"url":"/paper/satnet-bridging-deep-learning-and-logical","title":"SATNet: Bridging deep learning and logical reasoning using a differentiable satisfiability solver","date":"2019-05-29","arxiv_id":"1905.12149","repositories_listed":3,"syntology":{"n":5,"n_ran":4,"n_unverified":1,"n_pointer_only":0}},{"url":"/paper/a-dataset-and-architecture-for-visual","title":"A Dataset and Architecture for Visual Reasoning with a Working Memory","date":"2018-03-16","arxiv_id":"1803.06092","repositories_listed":3,"syntology":{"n":5,"n_ran":5,"n_unverified":0,"n_pointer_only":0}},{"url":"/paper/infigui-r1-advancing-multimodal-gui-agents","title":"InfiGUI-R1: Advancing Multimodal GUI Agents from Reactive Actors to Deliberative Reasoners","date":"2025-04-19","arxiv_id":"2504.14239","repositories_listed":2,"syntology":{"n":3,"n_ran":0,"n_unverified":3,"n_pointer_only":0}},{"url":"/paper/textgames-learning-to-self-play-text-based","title":"TextGames: Learning to Self-Play Text-Based Puzzle Games via Language Model Reasoning","date":"2025-02-25","arxiv_id":"2502.18431","repositories_listed":2,"syntology":{"n":3,"n_ran":2,"n_unverified":1,"n_pointer_only":3}},{"url":"/paper/enhancing-reasoning-capabilities-of-llms-via","title":"Enhancing Reasoning Capabilities of LLMs via Principled Synthetic Logic Corpus","date":"2024-11-19","arxiv_id":"2411.12498","repositories_listed":2,"syntology":null},{"url":"/paper/llava-o1-let-vision-language-models-reason","title":"LLaVA-CoT: Let Vision Language Models Reason Step-by-Step","date":"2024-11-15","arxiv_id":"2411.10440","repositories_listed":2,"syntology":{"n":6,"n_ran":0,"n_unverified":6,"n_pointer_only":0}},{"url":"/paper/gsm-symbolic-understanding-the-limitations-of","title":"GSM-Symbolic: Understanding the Limitations of Mathematical Reasoning in Large Language Models","date":"2024-10-07","arxiv_id":"2410.05229","repositories_listed":2,"syntology":null},{"url":"/paper/needlebench-can-llms-do-retrieval-and","title":"NeedleBench: Can LLMs Do Retrieval and Reasoning in Information-Dense Context?","date":"2024-07-16","arxiv_id":"2407.11963","repositories_listed":2,"syntology":null},{"url":"/paper/gtbench-uncovering-the-strategic-reasoning","title":"GTBench: Uncovering the Strategic Reasoning Limitations of LLMs via Game-Theoretic Evaluations","date":"2024-02-19","arxiv_id":"2402.12348","repositories_listed":2,"syntology":null},{"url":"/paper/efficiently-programming-large-language-models","title":"SGLang: Efficient Execution of Structured Language Model Programs","date":"2023-12-12","arxiv_id":"2312.07104","repositories_listed":2,"syntology":{"n":17,"n_ran":9,"n_unverified":8,"n_pointer_only":0}},{"url":"/paper/generating-by-understanding-neural-visual","title":"Generating by Understanding: Neural Visual Generation with Logical Symbol Groundings","date":"2023-10-26","arxiv_id":"2310.17451","repositories_listed":2,"syntology":null},{"url":"/paper/logicllm-exploring-self-supervised-logic","title":"Exploring Self-supervised Logic-enhanced Training for Large Language Models","date":"2023-05-23","arxiv_id":"2305.13718","repositories_listed":2,"syntology":{"n":8,"n_ran":2,"n_unverified":6,"n_pointer_only":0}},{"url":"/paper/polytuplet-loss-a-reverse-approach-to","title":"Deep Manifold Learning for Reading Comprehension and Logical Reasoning Tasks with Polytuplet Loss","date":"2023-04-03","arxiv_id":"2304.01046","repositories_listed":2,"syntology":null},{"url":"/paper/bloomberggpt-a-large-language-model-for","title":"BloombergGPT: A Large Language Model for Finance","date":"2023-03-30","arxiv_id":"2303.17564","repositories_listed":2,"syntology":null},{"url":"/paper/explicit-planning-helps-language-models-in","title":"Explicit Planning Helps Language Models in Logical Reasoning","date":"2023-03-28","arxiv_id":"2303.15714","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_unverified":0,"n_pointer_only":0}},{"url":"/paper/reasoning-with-language-model-prompting-a","title":"Reasoning with Language Model Prompting: A Survey","date":"2022-12-19","arxiv_id":"2212.09597","repositories_listed":2,"syntology":null},{"url":"/paper/unigeo-unifying-geometry-logical-reasoning","title":"UniGeo: Unifying Geometry Logical Reasoning via Reformulating Mathematical Expression","date":"2022-12-06","arxiv_id":"2212.02746","repositories_listed":2,"syntology":{"n":4,"n_ran":2,"n_unverified":2,"n_pointer_only":4}},{"url":"/paper/investigating-the-robustness-of-natural","title":"Investigating the Robustness of Natural Language Generation from Logical Forms via Counterfactual Samples","date":"2022-10-16","arxiv_id":"2210.08548","repositories_listed":2,"syntology":null}],"syntology_records":17,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-09-24T18:15:14+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"}},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"}}