{"url":"/sota/question-answering-on-openbookqa","task":{"name":"Question Answering","url":"/task/question-answering","note":null},"dataset":{"name":"OpenBookQA","url":"/dataset/openbookqa"},"category":"Natural Language Processing","categories":["Miscellaneous","Natural Language Processing","Reasoning"],"category_note":null,"description":"Question answering can be segmented into domain-specific tasks like community question answering and knowledge-base question answering. Popular benchmark datasets for evaluation question answering systems include [SQuAD](/dataset/squad), [HotPotQA](/dataset/hotpotqa), [bAbI](/dataset/babi-1), [TriviaQA](/dataset/triviaqa), [WikiQA](/dataset/wikiqa), and many others. Models for question answering are typically evaluated on metrics like EM and F1. Some recent top performing models are T5 and XLNet.\r\n\r\n( Image credit: [SQuAD](https://rajpurkar.github.io/mlx/qa-and-squad/) )","description_from":"task","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","rank":"the archive's row order at snapshot; not re-ranked","rows_end_at":"2025-07-28","rows_withheld_as_spam":0,"metric_values":"the archive's strings, untouched"},"metrics":["Accuracy"],"metric_direction":{"note":"inferred from the metric name only (the archive records no direction); null = not inferred, chart draws points only","by_metric":{"Accuracy":"higher"}},"counts":{"rows":45,"rows_with_code":29,"rows_with_paper_page":40,"rows_dated":40,"rows_using_additional_data":0},"rows":[{"rank_in_archive_order":1,"model":"GPT-4 + knowledge base","metrics":{"Accuracy":"95.9"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":2,"model":"MVP-Tuning (ensemble)","metrics":{"Accuracy":"95.2"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":3,"model":"PaLM 540B (Self Improvement, Self Consistency)","metrics":{"Accuracy":"94.4"},"uses_additional_data":false,"paper_date":"2022-10-20","paper":"/paper/large-language-models-can-self-improve","paper_url":"https://arxiv.org/abs/2210.11610v2","paper_title":"Large Language Models Can Self-Improve","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":4,"model":"X-Reasoner","metrics":{"Accuracy":"94.2"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":5,"model":"PaLM 540B (Self Improvement, CoT Prompting)","metrics":{"Accuracy":"93"},"uses_additional_data":false,"paper_date":"2022-10-20","paper":"/paper/large-language-models-can-self-improve","paper_url":"https://arxiv.org/abs/2210.11610v2","paper_title":"Large Language Models Can Self-Improve","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":6,"model":"PaLM 540B (Self Improvement, Standard-Prompting)","metrics":{"Accuracy":"92"},"uses_additional_data":false,"paper_date":"2022-10-20","paper":"/paper/large-language-models-can-self-improve","paper_url":"https://arxiv.org/abs/2210.11610v2","paper_title":"Large Language Models Can Self-Improve","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":7,"model":"DeBERTa-xxlarge 1.5B + MVP-Tuning","metrics":{"Accuracy":"91.3"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":8,"model":"PaLM 540B (Self Consistency)","metrics":{"Accuracy":"90"},"uses_additional_data":false,"paper_date":"2022-10-20","paper":"/paper/large-language-models-can-self-improve","paper_url":"https://arxiv.org/abs/2210.11610v2","paper_title":"Large Language Models Can Self-Improve","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":9,"model":"GrapeQA: PEGA+CANP","metrics":{"Accuracy":"90"},"uses_additional_data":false,"paper_date":"2023-03-22","paper":"/paper/grapeqa-graph-augmentation-and-pruning-to","paper_url":"https://arxiv.org/abs/2303.12320v2","paper_title":"GrapeQA: GRaph Augmentation and Pruning to Enhance Question-Answering","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":10,"model":"GenMC 11B","metrics":{"Accuracy":"89.8"},"uses_additional_data":false,"paper_date":"2022-04-30","paper":"/paper/clues-before-answers-generation-enhanced","paper_url":"https://arxiv.org/abs/2205.00274v1","paper_title":"Clues Before Answers: Generation-Enhanced Multiple-Choice QA","code":"https://github.com/nju-websoft/genmc","n_code_links":1,"syntology":{"n_ran":5,"n_unverified":6,"n_samples":11,"n_pointer_only_licence":0}},{"rank_in_archive_order":11,"model":"AristoRoBERTa + MVP-Tuning","metrics":{"Accuracy":"87.6"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":12,"model":"AristoRoBERTa + Graph Soft Counter","metrics":{"Accuracy":"87.4"},"uses_additional_data":false,"paper_date":"2021-10-07","paper":"/paper/gnn-is-a-counter-revisiting-gnn-for-question","paper_url":"https://arxiv.org/abs/2110.03192v1","paper_title":"GNN is a Counter? Revisiting GNN for Question Answering","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":13,"model":"UnifiedQA 11B","metrics":{"Accuracy":"87.2"},"uses_additional_data":false,"paper_date":"2020-05-02","paper":"/paper/unifiedqa-crossing-format-boundaries-with-a","paper_url":"https://arxiv.org/abs/2005.00700v3","paper_title":"UnifiedQA: Crossing Format Boundaries With a Single QA System","code":"https://github.com/allenai/unifiedqa","n_code_links":2,"syntology":{"n_ran":4,"n_unverified":3,"n_samples":7,"n_pointer_only_licence":1}},{"rank_in_archive_order":14,"model":"LLaMA-3 8B+MoSLoRA","metrics":{"Accuracy":"86.8"},"uses_additional_data":false,"paper_date":"2024-06-16","paper":"/paper/mixture-of-subspaces-in-low-rank-adaptation","paper_url":"https://arxiv.org/abs/2406.11909v3","paper_title":"Mixture-of-Subspaces in Low-Rank Adaptation","code":"https://github.com/wutaiqiang/moslora","n_code_links":1,"syntology":{"n_ran":4,"n_unverified":2,"n_samples":6,"n_pointer_only_licence":6}},{"rank_in_archive_order":15,"model":"PaLM 540B (CoT Prompting)","metrics":{"Accuracy":"86.4"},"uses_additional_data":false,"paper_date":"2022-10-20","paper":"/paper/large-language-models-can-self-improve","paper_url":"https://arxiv.org/abs/2210.11610v2","paper_title":"Large Language Models Can Self-Improve","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":16,"model":"LLaMA-3 8B + MixLoRA","metrics":{"Accuracy":"84.8"},"uses_additional_data":false,"paper_date":"2024-04-22","paper":"/paper/mixlora-enhancing-large-language-models-fine","paper_url":"https://arxiv.org/abs/2404.15159v3","paper_title":"MixLoRA: Enhancing Large Language Models Fine-Tuning with LoRA-based Mixture of Experts","code":"https://github.com/TUDB-Labs/MixLoRA","n_code_links":2,"syntology":{"n_ran":6,"n_unverified":5,"n_samples":11,"n_pointer_only_licence":0}},{"rank_in_archive_order":17,"model":"PaLM 540B (Standard-Prompting)","metrics":{"Accuracy":"84.4"},"uses_additional_data":false,"paper_date":"2022-10-20","paper":"/paper/large-language-models-can-self-improve","paper_url":"https://arxiv.org/abs/2210.11610v2","paper_title":"Large Language Models Can Self-Improve","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":18,"model":"TTTTT 3B","metrics":{"Accuracy":"83.2"},"uses_additional_data":false,"paper_date":"2020-12-09","paper":"/paper/fusing-context-into-knowledge-graph-for","paper_url":"https://arxiv.org/abs/2012.04808v3","paper_title":"Fusing Context Into Knowledge Graph for Commonsense Question Answering","code":"https://github.com/microsoft/kear","n_code_links":2,"syntology":null},{"rank_in_archive_order":19,"model":"LLaMA-2 13B + MixLoRA","metrics":{"Accuracy":"83"},"uses_additional_data":false,"paper_date":"2024-04-22","paper":"/paper/mixlora-enhancing-large-language-models-fine","paper_url":"https://arxiv.org/abs/2404.15159v3","paper_title":"MixLoRA: Enhancing Large Language Models Fine-Tuning with LoRA-based Mixture of Experts","code":"https://github.com/TUDB-Labs/MixLoRA","n_code_links":2,"syntology":{"n_ran":6,"n_unverified":5,"n_samples":11,"n_pointer_only_licence":0}},{"rank_in_archive_order":20,"model":"AristoRoBERTa + QA-GNN","metrics":{"Accuracy":"82.8"},"uses_additional_data":false,"paper_date":"2021-04-13","paper":"/paper/qa-gnn-reasoning-with-language-models-and","paper_url":"https://arxiv.org/abs/2104.06378v5","paper_title":"QA-GNN: Reasoning with Language Models and Knowledge Graphs for Question Answering","code":"https://github.com/michiyasunaga/qagnn","n_code_links":6,"syntology":{"n_ran":1,"n_unverified":24,"n_samples":25,"n_pointer_only_licence":4}},{"rank_in_archive_order":21,"model":"QA-GNN","metrics":{"Accuracy":"82.8"},"uses_additional_data":false,"paper_date":"2021-04-13","paper":"/paper/qa-gnn-reasoning-with-language-models-and","paper_url":"https://arxiv.org/abs/2104.06378v5","paper_title":"QA-GNN: Reasoning with Language Models and Knowledge Graphs for Question Answering","code":"https://github.com/michiyasunaga/qagnn","n_code_links":6,"syntology":{"n_ran":1,"n_unverified":24,"n_samples":25,"n_pointer_only_licence":4}},{"rank_in_archive_order":22,"model":"DEKCOR","metrics":{"Accuracy":"82.4"},"uses_additional_data":false,"paper_date":"2020-12-09","paper":"/paper/fusing-context-into-knowledge-graph-for","paper_url":"https://arxiv.org/abs/2012.04808v3","paper_title":"Fusing Context Into Knowledge Graph for Commonsense Question Answering","code":"https://github.com/microsoft/kear","n_code_links":2,"syntology":null},{"rank_in_archive_order":23,"model":"GrapeQA: PEGA","metrics":{"Accuracy":"82"},"uses_additional_data":false,"paper_date":"2023-03-22","paper":"/paper/grapeqa-graph-augmentation-and-pruning-to","paper_url":"https://arxiv.org/abs/2303.12320v2","paper_title":"GrapeQA: GRaph Augmentation and Pruning to Enhance Question-Answering","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":24,"model":"LLaMA-2 7B + MixLoRA","metrics":{"Accuracy":"81.6"},"uses_additional_data":false,"paper_date":"2024-04-22","paper":"/paper/mixlora-enhancing-large-language-models-fine","paper_url":"https://arxiv.org/abs/2404.15159v3","paper_title":"MixLoRA: Enhancing Large Language Models Fine-Tuning with LoRA-based Mixture of Experts","code":"https://github.com/TUDB-Labs/MixLoRA","n_code_links":2,"syntology":{"n_ran":6,"n_unverified":5,"n_samples":11,"n_pointer_only_licence":0}},{"rank_in_archive_order":25,"model":"AristoRoBERTa","metrics":{"Accuracy":"77.8"},"uses_additional_data":false,"paper_date":"2021-04-13","paper":"/paper/qa-gnn-reasoning-with-language-models-and","paper_url":"https://arxiv.org/abs/2104.06378v5","paper_title":"QA-GNN: Reasoning with Language Models and Knowledge Graphs for Question Answering","code":"https://github.com/michiyasunaga/qagnn","n_code_links":6,"syntology":{"n_ran":1,"n_unverified":24,"n_samples":25,"n_pointer_only_licence":4}},{"rank_in_archive_order":26,"model":"BiLSTM max-out question-match (science fact + common knowledge fact)","metrics":{"Accuracy":"76.9"},"uses_additional_data":false,"paper_date":"2018-09-08","paper":"/paper/can-a-suit-of-armor-conduct-electricity-a-new","paper_url":"http://arxiv.org/abs/1809.02789v1","paper_title":"Can a Suit of Armor Conduct Electricity? A New Dataset for Open Book Question Answering","code":"https://github.com/allenai/arc-solvers","n_code_links":1,"syntology":{"n_ran":0,"n_unverified":3,"n_samples":3,"n_pointer_only_licence":0}},{"rank_in_archive_order":27,"model":"Careful Selection","metrics":{"Accuracy":"72"},"uses_additional_data":false,"paper_date":"2019-07-24","paper":"/paper/careful-selection-of-knowledge-to-solve-open","paper_url":"https://arxiv.org/abs/1907.10738v1","paper_title":"Careful Selection of Knowledge to solve Open Book Question Answering","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":28,"model":"GrapeQA: CANP","metrics":{"Accuracy":"66.2"},"uses_additional_data":false,"paper_date":"2023-03-22","paper":"/paper/grapeqa-graph-augmentation-and-pruning-to","paper_url":"https://arxiv.org/abs/2303.12320v2","paper_title":"GrapeQA: GRaph Augmentation and Pruning to Enhance Question-Answering","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":29,"model":"GPT-3 175B (few-shot, k=32)","metrics":{"Accuracy":"65.4"},"uses_additional_data":false,"paper_date":"2020-05-28","paper":"/paper/language-models-are-few-shot-learners","paper_url":"https://arxiv.org/abs/2005.14165v4","paper_title":"Language Models are Few-Shot Learners","code":"https://github.com/ggml-org/llama.cpp","n_code_links":67,"syntology":{"n_ran":15,"n_unverified":50,"n_samples":65,"n_pointer_only_licence":4}},{"rank_in_archive_order":30,"model":"PaLM 2-L (1-shot)","metrics":{"Accuracy":"58.5"},"uses_additional_data":false,"paper_date":"2023-05-17","paper":"/paper/palm-2-technical-report-1","paper_url":"https://arxiv.org/abs/2305.10403v3","paper_title":"PaLM 2 Technical Report","code":"https://github.com/eternityyw/tram-benchmark","n_code_links":1,"syntology":null},{"rank_in_archive_order":31,"model":"OPT 66B (one-shot)","metrics":{"Accuracy":"58.0"},"uses_additional_data":false,"paper_date":"2023-03-30","paper":"/paper/bloomberggpt-a-large-language-model-for","paper_url":"https://arxiv.org/abs/2303.17564v3","paper_title":"BloombergGPT: A Large Language Model for Finance","code":"https://github.com/yangletliu/finlora","n_code_links":2,"syntology":null},{"rank_in_archive_order":32,"model":"PaLM 2-S (1-shot)","metrics":{"Accuracy":"57.4"},"uses_additional_data":false,"paper_date":"2023-05-17","paper":"/paper/palm-2-technical-report-1","paper_url":"https://arxiv.org/abs/2305.10403v3","paper_title":"PaLM 2 Technical Report","code":"https://github.com/eternityyw/tram-benchmark","n_code_links":1,"syntology":null},{"rank_in_archive_order":33,"model":"BiLSTM max-out question-match (WordNet + science fact)","metrics":{"Accuracy":"56.3"},"uses_additional_data":false,"paper_date":"2018-09-08","paper":"/paper/can-a-suit-of-armor-conduct-electricity-a-new","paper_url":"http://arxiv.org/abs/1809.02789v1","paper_title":"Can a Suit of Armor Conduct Electricity? A New Dataset for Open Book Question Answering","code":"https://github.com/allenai/arc-solvers","n_code_links":1,"syntology":{"n_ran":0,"n_unverified":3,"n_samples":3,"n_pointer_only_licence":0}},{"rank_in_archive_order":34,"model":"PaLM 2-M (1-shot)","metrics":{"Accuracy":"56.2"},"uses_additional_data":false,"paper_date":"2023-05-17","paper":"/paper/palm-2-technical-report-1","paper_url":"https://arxiv.org/abs/2305.10403v3","paper_title":"PaLM 2 Technical Report","code":"https://github.com/eternityyw/tram-benchmark","n_code_links":1,"syntology":null},{"rank_in_archive_order":35,"model":"BiLSTM max-out question-match (with a science fact)","metrics":{"Accuracy":"55.8"},"uses_additional_data":false,"paper_date":"2018-09-08","paper":"/paper/can-a-suit-of-armor-conduct-electricity-a-new","paper_url":"http://arxiv.org/abs/1809.02789v1","paper_title":"Can a Suit of Armor Conduct Electricity? A New Dataset for Open Book Question Answering","code":"https://github.com/allenai/arc-solvers","n_code_links":1,"syntology":{"n_ran":0,"n_unverified":3,"n_samples":3,"n_pointer_only_licence":0}},{"rank_in_archive_order":36,"model":"Bloomberg GPT 50B (1-shot)","metrics":{"Accuracy":"51.6"},"uses_additional_data":false,"paper_date":"2023-03-30","paper":"/paper/bloomberggpt-a-large-language-model-for","paper_url":"https://arxiv.org/abs/2303.17564v3","paper_title":"BloombergGPT: A Large Language Model for Finance","code":"https://github.com/yangletliu/finlora","n_code_links":2,"syntology":null},{"rank_in_archive_order":37,"model":"BLOOM 176B (2-shot)","metrics":{"Accuracy":"47.2"},"uses_additional_data":false,"paper_date":"2023-03-30","paper":"/paper/bloomberggpt-a-large-language-model-for","paper_url":"https://arxiv.org/abs/2303.17564v3","paper_title":"BloombergGPT: A Large Language Model for Finance","code":"https://github.com/yangletliu/finlora","n_code_links":2,"syntology":null},{"rank_in_archive_order":38,"model":"GPT-NeoX 50B (2-shot)","metrics":{"Accuracy":"44.2"},"uses_additional_data":false,"paper_date":"2023-03-30","paper":"/paper/bloomberggpt-a-large-language-model-for","paper_url":"https://arxiv.org/abs/2303.17564v3","paper_title":"BloombergGPT: A Large Language Model for Finance","code":"https://github.com/yangletliu/finlora","n_code_links":2,"syntology":null},{"rank_in_archive_order":39,"model":"LaMini-GPT 1.5B","metrics":{"Accuracy":"39.8"},"uses_additional_data":false,"paper_date":"2023-04-27","paper":"/paper/lamini-lm-a-diverse-herd-of-distilled-models","paper_url":"https://arxiv.org/abs/2304.14402v3","paper_title":"LaMini-LM: A Diverse Herd of Distilled Models from Large-Scale Instructions","code":"https://github.com/mbzuai-nlp/lamini-lm","n_code_links":1,"syntology":null},{"rank_in_archive_order":40,"model":"LaMini-T5 738M","metrics":{"Accuracy":"36"},"uses_additional_data":false,"paper_date":"2023-04-27","paper":"/paper/lamini-lm-a-diverse-herd-of-distilled-models","paper_url":"https://arxiv.org/abs/2304.14402v3","paper_title":"LaMini-LM: A Diverse Herd of Distilled Models from Large-Scale Instructions","code":"https://github.com/mbzuai-nlp/lamini-lm","n_code_links":1,"syntology":null},{"rank_in_archive_order":41,"model":"LaMini-F-T5 783M","metrics":{"Accuracy":"34"},"uses_additional_data":false,"paper_date":"2023-04-27","paper":"/paper/lamini-lm-a-diverse-herd-of-distilled-models","paper_url":"https://arxiv.org/abs/2304.14402v3","paper_title":"LaMini-LM: A Diverse Herd of Distilled Models from Large-Scale Instructions","code":"https://github.com/mbzuai-nlp/lamini-lm","n_code_links":1,"syntology":null},{"rank_in_archive_order":42,"model":"T5-Large 738M","metrics":{"Accuracy":"32.8"},"uses_additional_data":false,"paper_date":"2023-04-27","paper":"/paper/lamini-lm-a-diverse-herd-of-distilled-models","paper_url":"https://arxiv.org/abs/2304.14402v3","paper_title":"LaMini-LM: A Diverse Herd of Distilled Models from Large-Scale Instructions","code":"https://github.com/mbzuai-nlp/lamini-lm","n_code_links":1,"syntology":null},{"rank_in_archive_order":43,"model":"GPT-2-XL 1.5B","metrics":{"Accuracy":"32"},"uses_additional_data":false,"paper_date":"2023-04-27","paper":"/paper/lamini-lm-a-diverse-herd-of-distilled-models","paper_url":"https://arxiv.org/abs/2304.14402v3","paper_title":"LaMini-LM: A Diverse Herd of Distilled Models from Large-Scale Instructions","code":"https://github.com/mbzuai-nlp/lamini-lm","n_code_links":1,"syntology":null},{"rank_in_archive_order":44,"model":"FLAN-T5-Large 783M","metrics":{"Accuracy":"31.2"},"uses_additional_data":false,"paper_date":"2023-04-27","paper":"/paper/lamini-lm-a-diverse-herd-of-distilled-models","paper_url":"https://arxiv.org/abs/2304.14402v3","paper_title":"LaMini-LM: A Diverse Herd of Distilled Models from Large-Scale Instructions","code":"https://github.com/mbzuai-nlp/lamini-lm","n_code_links":1,"syntology":null},{"rank_in_archive_order":45,"model":"Random chance baseline","metrics":{"Accuracy":"25"},"uses_additional_data":false,"paper_date":"2018-09-08","paper":"/paper/can-a-suit-of-armor-conduct-electricity-a-new","paper_url":"http://arxiv.org/abs/1809.02789v1","paper_title":"Can a Suit of Armor Conduct Electricity? A New Dataset for Open Book Question Answering","code":"https://github.com/allenai/arc-solvers","n_code_links":1,"syntology":{"n_ran":0,"n_unverified":3,"n_samples":3,"n_pointer_only_licence":0}}],"since_archive":{"present":false,"note":"No Syntology-extracted rows are published in this build."},"syntology":{"read_at":"2026-09-24T18:15:14+00:00","claim":"Per row: N of M harvested code samples from that row's paper executed on a synthesized fixture; the other M-N are unverified. Not a reproduction of the row's number; not a correctness claim. n_pointer_only_licence counts samples the site points at rather than redistributes (a licence axis, independent of ran/unverified).","rows_with_graph_line":14,"rows_with_any_sample_ran":10,"distinct_papers_with_graph_line":7,"distinct_papers_with_any_sample_ran":6,"samples_over_distinct_papers":{"n_ran":35,"n_unverified":93,"n_samples":128,"n_pointer_only_licence":15,"note":"each paper (arXiv id) counted once, however many rows it is behind; this is the page-level figure"},"samples_row_weighted":{"n_ran":49,"n_unverified":160,"n_samples":209,"n_pointer_only_licence":23,"note":"row-weighted: a paper behind several rows is counted once per row; inflated relative to samples_over_distinct_papers by design, kept for readers summing the per-row syntology blocks"}}}