{"url":"/dataset/natural-questions","name":"Natural Questions","full_name":null,"description_markdown":"The **Natural Questions** corpus is a question answering dataset containing 307,373 training examples, 7,830 development examples, and 7,842 test examples. Each example is comprised of a google.com query and a corresponding Wikipedia page. Each Wikipedia page has a passage (or long answer) annotated on the page that answers the question and one or more short spans from the annotated passage containing the actual answer. The long and the short answer annotations can however be empty. If they are both empty, then there is no answer on the page at all. If the long answer annotation is non-empty, but the short answer annotation is empty, then the annotated passage answers the question but no explicit short answer could be found. Finally 1% of the documents have a passage annotated with a short answer that is “yes” or “no”, instead of a list of short spans.\r\n\r\nSource: [A BERT Baseline for the Natural Questions](https://arxiv.org/abs/1901.08634)\r\nImage Source: [https://paperswithcode.com/paper/natural-questions-a-benchmark-for-question/](https://paperswithcode.com/paper/natural-questions-a-benchmark-for-question/)","description_withheld":null,"homepage":"https://ai.google.com/research/NaturalQuestions","introduced_date":"2019-01-01","introduced_date_note":null,"introduced_by":{"paper":"/paper/natural-questions-a-benchmark-for-question","title":"Natural Questions: a Benchmark for Question Answering Research","first_author":"Tom Kwiatkowski","url":null},"license":{"name":"CC BY-SA 3.0","url":"https://creativecommons.org/licenses/by-sa/3.0/"},"modalities":[{"name":"Texts","url":"/datasets/modality/texts"}],"tasks":[{"name":"Question Answering","url":"/task/question-answering","datasets_with_task":"/datasets/task/question-answering"},{"name":"Retrieval","url":"/task/retrieval","datasets_with_task":"/datasets/task/retrieval"},{"name":"Passage Retrieval","url":"/task/passage-retrieval","datasets_with_task":"/datasets/task/passage-retrieval"},{"name":"Zero-shot Text Search","url":"/task/zero-shot-text-search","datasets_with_task":"/datasets/task/zero-shot-text-search"},{"name":"Text Retrieval","url":"/task/text-retrieval","datasets_with_task":"/datasets/task/text-retrieval"},{"name":"Open-Domain Question Answering","url":"/task/open-domain-question-answering","datasets_with_task":"/datasets/task/open-domain-question-answering"},{"name":"Question Generation","url":"/task/question-generation","datasets_with_task":"/datasets/task/question-generation"}],"languages":[{"name":"English","url":"/datasets/language/english"}],"variants":["Natural Questions","Natural Questions (long)","Natural Questions (short)","NQ","NQ (BEIR)"],"data_loaders":[{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/rongzhangibm/NaturalQuestionsV2","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/natural_questions","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/KaraKaraWitch/NextGenBench","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/google-research-datasets/natural_questions","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/facebookresearch/ParlAI","url":"https://parl.ai/docs/tasks.html#natural-questions","frameworks":["pytorch"]},{"repo":"https://github.com/tensorflow/datasets","url":"https://www.tensorflow.org/datasets/catalog/natural_questions","frameworks":["tf","jax"]},{"repo":"https://github.com/RUCAIBox/LLMBox","url":"https://github.com/RUCAIBox/LLMBox/blob/main/docs/utilization/supported-datasets.md","frameworks":["pytorch"]}],"num_papers_in_archive":1404,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28"},"benchmarks":[{"leaderboard":"/sota/question-answering-on-natural-questions","task":"Question Answering","dataset_variant":"Natural Questions","rows":47,"metrics":["EM"],"first_row_in_archive_order":{"model":"Atlas (full, Wiki-dec-2018 index)","paper":"/paper/few-shot-learning-with-retrieval-augmented","metrics":{"EM":"64.0"},"code_links":[{"title":"facebookresearch/atlas","url":"https://github.com/facebookresearch/atlas"},{"title":"thunlp/clueanchor","url":"https://github.com/thunlp/clueanchor"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/question-answering-on-natural-questions-long","task":"Question Answering","dataset_variant":"Natural Questions (long)","rows":13,"metrics":["F1","EM"],"first_row_in_archive_order":{"model":"DensePhrases","paper":"/paper/learning-dense-representations-of-phrases-at","metrics":{"EM":"71.9","F1":"79.6"},"code_links":[{"title":"princeton-nlp/SimCSE","url":"https://github.com/princeton-nlp/SimCSE"},{"title":"jhyuklee/DensePhrases","url":"https://github.com/jhyuklee/DensePhrases"},{"title":"princeton-nlp/DensePhrases","url":"https://github.com/princeton-nlp/DensePhrases"},{"title":"dmis-lab/gener","url":"https://github.com/dmis-lab/gener"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/passage-retrieval-on-natural-questions","task":"Passage Retrieval","dataset_variant":"Natural Questions","rows":10,"metrics":["Precision@100","Precision@20"],"first_row_in_archive_order":{"model":"ReAtt","paper":"/paper/retrieval-as-attention-end-to-end-learning-of","metrics":{"Precision@100":"90.40","Precision@20":"86.00"},"code_links":[{"title":"jzbjyb/reatt","url":"https://github.com/jzbjyb/reatt"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/question-answering-on-nq-beir","task":"Question Answering","dataset_variant":"NQ (BEIR)","rows":6,"metrics":["nDCG@10"],"first_row_in_archive_order":{"model":"Blended RAG","paper":"/paper/blended-rag-improving-rag-retriever-augmented","metrics":{"nDCG@10":"0.67"},"code_links":[{"title":"ibm-ecosystem-engineering/blended-rag","url":"https://github.com/ibm-ecosystem-engineering/blended-rag"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/open-domain-question-answering-on-natural","task":"Open-Domain Question Answering","dataset_variant":"Natural Questions","rows":5,"metrics":["Exact Match"],"first_row_in_archive_order":{"model":"FiE","paper":"/paper/0-8-nyquist-computational-ghost-imaging-via","metrics":{"Exact Match":"58.4"},"code_links":[]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/retrieval-on-natural-questions","task":"Retrieval","dataset_variant":"Natural Questions","rows":3,"metrics":["Queries per second"],"first_row_in_archive_order":{"model":"BM25S","paper":"/paper/bm25s-orders-of-magnitude-faster-lexical","metrics":{"Queries per second":"41.85"},"code_links":[{"title":"xhluca/bm25s","url":"https://github.com/xhluca/bm25s"},{"title":"xhluca/bm25-benchmarks","url":"https://github.com/xhluca/bm25-benchmarks"},{"title":"conda-forge/bm25s-feedstock","url":"https://github.com/conda-forge/bm25s-feedstock"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/question-generation-on-natural-questions","task":"Question Generation","dataset_variant":"Natural Questions","rows":2,"metrics":["QAE","R-QAE"],"first_row_in_archive_order":{"model":"Info-HCVAE","paper":"/paper/generating-diverse-and-consistent-qa-pairs","metrics":{"QAE":"37.18","R-QAE":"29.39"},"code_links":[{"title":"seanie12/Info-HCVAE","url":"https://github.com/seanie12/Info-HCVAE"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/open-domain-question-answering-on-natural-1","task":"Open-Domain Question Answering","dataset_variant":"Natural Questions (short)","rows":1,"metrics":["Exact Match"],"first_row_in_archive_order":{"model":"EMDR2","paper":"/paper/end-to-end-training-of-multi-document-reader","metrics":{"Exact Match":"52.5"},"code_links":[{"title":"DevSinghSachan/emdr2","url":"https://github.com/DevSinghSachan/emdr2"},{"title":"DevSinghSachan/art","url":"https://github.com/DevSinghSachan/art"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/text-retrieval-on-natural-questions","task":"Text Retrieval","dataset_variant":"Natural Questions","rows":1,"metrics":["NDCG@10"],"first_row_in_archive_order":{"model":"Lucene (BM25S)","paper":"/paper/bm25s-orders-of-magnitude-faster-lexical","metrics":{"NDCG@10":"30.5"},"code_links":[{"title":"xhluca/bm25s","url":"https://github.com/xhluca/bm25s"},{"title":"xhluca/bm25-benchmarks","url":"https://github.com/xhluca/bm25-benchmarks"},{"title":"conda-forge/bm25s-feedstock","url":"https://github.com/conda-forge/bm25s-feedstock"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"}],"papers_with_a_benchmark_row":[{"paper":"/paper/search-o1-agentic-search-enhanced-large","title":"Search-o1: Agentic Search-Enhanced Large Reasoning Models","date":"2025-01-09","rows_on_this_dataset":1,"code_links":2,"syntology":null},{"paper":"/paper/bm25s-orders-of-magnitude-faster-lexical","title":"BM25S: Orders of magnitude faster lexical search via eager sparse scoring","date":"2024-07-04","rows_on_this_dataset":4,"code_links":3,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":22,"samples_ran":10,"samples_unverified":12,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/rankrag-unifying-context-ranking-with","title":"RankRAG: Unifying Context Ranking with Retrieval-Augmented Generation in LLMs","date":"2024-07-02","rows_on_this_dataset":4,"code_links":0,"syntology":null},{"paper":"/paper/understand-what-llm-needs-dual-preference","title":"Understand What LLM Needs: Dual Preference Alignment for Retrieval-Augmented Generation","date":"2024-06-26","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":3,"samples_ran":1,"samples_unverified":2,"pointer_only_for_licence":3,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/blended-rag-improving-rag-retriever-augmented","title":"Blended RAG: Improving RAG (Retriever-Augmented Generation) Accuracy with Semantic Search and Hybrid Query-Based Retrievers","date":"2024-03-22","rows_on_this_dataset":2,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":2,"samples_ran":1,"samples_unverified":1,"pointer_only_for_licence":2,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/chatqa-building-gpt-4-level-conversational-qa","title":"ChatQA: Surpassing GPT-4 on Conversational QA and RAG","date":"2024-01-18","rows_on_this_dataset":2,"code_links":0,"syntology":null},{"paper":"/paper/mistral-7b","title":"Mistral 7B","date":"2023-10-10","rows_on_this_dataset":1,"code_links":6,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":11,"samples_ran":9,"samples_unverified":2,"pointer_only_for_licence":1,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/llama-2-open-foundation-and-fine-tuned-chat","title":"Llama 2: Open Foundation and Fine-Tuned Chat Models","date":"2023-07-18","rows_on_this_dataset":1,"code_links":19,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":52,"samples_ran":31,"samples_unverified":21,"pointer_only_for_licence":16,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/palm-2-technical-report-1","title":"PaLM 2 Technical Report","date":"2023-05-17","rows_on_this_dataset":3,"code_links":1,"syntology":null},{"paper":"/paper/llama-open-and-efficient-foundation-language-1","title":"LLaMA: Open and Efficient Foundation Language Models","date":"2023-02-27","rows_on_this_dataset":4,"code_links":57,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":58,"samples_ran":26,"samples_unverified":32,"pointer_only_for_licence":4,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/replug-retrieval-augmented-black-box-language","title":"REPLUG: Retrieval-Augmented Black-Box Language Models","date":"2023-01-30","rows_on_this_dataset":2,"code_links":3,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":13,"samples_ran":0,"samples_unverified":13,"pointer_only_for_licence":13,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/retrieval-as-attention-end-to-end-learning-of","title":"Retrieval as Attention: End-to-end Learning of Retrieval and Reading within a Single Transformer","date":"2022-12-05","rows_on_this_dataset":2,"code_links":1,"syntology":null},{"paper":"/paper/fie-building-a-global-probability-space-by","title":"FiE: Building a Global Probability Space by Leveraging Early Fusion in Encoder for Open-Domain Question Answering","date":"2022-11-18","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/ask-me-anything-a-simple-strategy-for","title":"Ask Me Anything: A simple strategy for prompting language models","date":"2022-10-05","rows_on_this_dataset":3,"code_links":3,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":2,"samples_ran":2,"samples_unverified":0,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/few-shot-learning-with-retrieval-augmented","title":"Atlas: Few-shot Learning with Retrieval Augmented Language Models","date":"2022-08-05","rows_on_this_dataset":4,"code_links":2,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":2,"samples_ran":2,"samples_unverified":0,"pointer_only_for_licence":2,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/no-parameter-left-behind-how-distillation-and","title":"No Parameter Left Behind: How Distillation and Model Size Affect Zero-Shot Retrieval","date":"2022-06-06","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/palm-scaling-language-modeling-with-pathways-1","title":"PaLM: Scaling Language Modeling with Pathways","date":"2022-04-05","rows_on_this_dataset":3,"code_links":7,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":37,"samples_ran":30,"samples_unverified":7,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/training-compute-optimal-large-language","title":"Training Compute-Optimal Large Language Models","date":"2022-03-29","rows_on_this_dataset":1,"code_links":2,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":11,"samples_ran":8,"samples_unverified":3,"pointer_only_for_licence":4,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/augmenting-document-representations-for-dense-1","title":"Augmenting Document Representations for Dense Retrieval with Interpolation and Perturbation","date":"2022-03-15","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/sgpt-gpt-sentence-embeddings-for-semantic","title":"SGPT: GPT Sentence Embeddings for Semantic Search","date":"2022-02-17","rows_on_this_dataset":2,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":1,"samples_ran":1,"samples_unverified":0,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/glam-efficient-scaling-of-language-models","title":"GLaM: Efficient Scaling of Language Models with Mixture-of-Experts","date":"2021-12-13","rows_on_this_dataset":3,"code_links":0,"syntology":null},{"paper":"/paper/scaling-language-models-methods-analysis-1","title":"Scaling Language Models: Methods, Analysis & Insights from Training Gopher","date":"2021-12-08","rows_on_this_dataset":1,"code_links":3,"syntology":null},{"paper":"/paper/improving-language-models-by-retrieving-from","title":"Improving language models by retrieving from trillions of tokens","date":"2021-12-08","rows_on_this_dataset":1,"code_links":2,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":23,"samples_ran":16,"samples_unverified":7,"pointer_only_for_licence":3,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/salient-phrase-aware-dense-retrieval-can-a","title":"Salient Phrase Aware Dense Retrieval: Can a Dense Retriever Imitate a Sparse One?","date":"2021-10-13","rows_on_this_dataset":1,"code_links":2,"syntology":null},{"paper":"/paper/r2-d2-a-modular-baseline-for-open-domain","title":"R2-D2: A Modular Baseline for Open-Domain Question Answering","date":"2021-09-08","rows_on_this_dataset":5,"code_links":1,"syntology":null},{"paper":"/paper/0-8-nyquist-computational-ghost-imaging-via","title":"0.8% Nyquist computational ghost imaging via non-experimental deep learning","date":"2021-08-17","rows_on_this_dataset":2,"code_links":0,"syntology":null},{"paper":"/paper/domain-matched-pre-training-tasks-for-dense","title":"Domain-matched Pre-training Tasks for Dense Retrieval","date":"2021-07-28","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/end-to-end-training-of-multi-document-reader","title":"End-to-End Training of Multi-Document Reader and Retriever for Open-Domain Question Answering","date":"2021-06-09","rows_on_this_dataset":2,"code_links":2,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":4,"samples_ran":4,"samples_unverified":0,"pointer_only_for_licence":4,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/efficient-passage-retrieval-with-hashing-for","title":"Efficient Passage Retrieval with Hashing for Open-domain Question Answering","date":"2021-06-02","rows_on_this_dataset":2,"code_links":1,"syntology":null},{"paper":"/paper/beir-a-heterogenous-benchmark-for-zero-shot","title":"BEIR: A Heterogenous Benchmark for Zero-shot Evaluation of Information Retrieval Models","date":"2021-04-17","rows_on_this_dataset":2,"code_links":3,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":4,"samples_ran":3,"samples_unverified":1,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/unitedqa-a-hybrid-approach-for-open-domain","title":"UnitedQA: A Hybrid Approach for Open Domain Question Answering","date":"2021-01-01","rows_on_this_dataset":2,"code_links":0,"syntology":null},{"paper":"/paper/unified-open-domain-question-answering-with","title":"UniK-QA: Unified Representations of Structured and Unstructured Knowledge for Open-Domain Question Answering","date":"2020-12-29","rows_on_this_dataset":2,"code_links":1,"syntology":null},{"paper":"/paper/learning-dense-representations-of-phrases-at","title":"Learning Dense Representations of Phrases at Scale","date":"2020-12-23","rows_on_this_dataset":1,"code_links":4,"syntology":null},{"paper":"/paper/rocketqa-an-optimized-training-approach-to","title":"RocketQA: An Optimized Training Approach to Dense Passage Retrieval for Open-Domain Question Answering","date":"2020-10-16","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/generation-augmented-retrieval-for-open","title":"Generation-Augmented Retrieval for Open-domain Question Answering","date":"2020-09-17","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/cluster-former-clustering-based-sparse","title":"Cluster-Former: Clustering-based Sparse Transformer for Long-Range Dependency Encoding","date":"2020-09-13","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/leveraging-passage-retrieval-with-generative","title":"Leveraging Passage Retrieval with Generative Models for Open Domain Question Answering","date":"2020-07-02","rows_on_this_dataset":2,"code_links":8,"syntology":null},{"paper":"/paper/approximate-nearest-neighbor-negative","title":"Approximate Nearest Neighbor Negative Contrastive Learning for Dense Text Retrieval","date":"2020-07-01","rows_on_this_dataset":1,"code_links":5,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":12,"samples_ran":3,"samples_unverified":9,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/language-models-are-few-shot-learners","title":"Language Models are Few-Shot Learners","date":"2020-05-28","rows_on_this_dataset":1,"code_links":67,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":65,"samples_ran":15,"samples_unverified":50,"pointer_only_for_licence":4,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/generating-diverse-and-consistent-qa-pairs","title":"Generating Diverse and Consistent QA pairs from Contexts with Information-Maximizing Hierarchical Conditional VAEs","date":"2020-05-28","rows_on_this_dataset":2,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":16,"samples_ran":0,"samples_unverified":16,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/retrieval-augmented-generation-for-knowledge","title":"Retrieval-Augmented Generation for Knowledge-Intensive NLP Tasks","date":"2020-05-22","rows_on_this_dataset":1,"code_links":18,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":6,"samples_ran":4,"samples_unverified":2,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/dense-passage-retrieval-for-open-domain","title":"Dense Passage Retrieval for Open-Domain Question Answering","date":"2020-04-10","rows_on_this_dataset":2,"code_links":19,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":14,"samples_ran":10,"samples_unverified":4,"pointer_only_for_licence":9,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/realm-retrieval-augmented-language-model-pre","title":"REALM: Retrieval-Augmented Language Model Pre-Training","date":"2020-02-10","rows_on_this_dataset":1,"code_links":6,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":4,"samples_ran":4,"samples_unverified":0,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/reformer-the-efficient-transformer-1","title":"Reformer: The Efficient Transformer","date":"2020-01-13","rows_on_this_dataset":1,"code_links":10,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":8,"samples_ran":6,"samples_unverified":2,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/frustratingly-easy-natural-question-answering","title":"Frustratingly Easy Natural Question Answering","date":"2019-09-11","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/natural-questions-a-benchmark-for-question","title":"Natural Questions: a Benchmark for Question Answering Research","date":"2019-06-01","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/190410509","title":"Generating Long Sequences with Sparse Transformers","date":"2019-04-23","rows_on_this_dataset":1,"code_links":7,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":6,"samples_ran":5,"samples_unverified":1,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/a-bert-baseline-for-the-natural-questions","title":"A BERT Baseline for the Natural Questions","date":"2019-01-24","rows_on_this_dataset":1,"code_links":3,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":10,"samples_ran":0,"samples_unverified":10,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/reading-wikipedia-to-answer-open-domain","title":"Reading Wikipedia to Answer Open-Domain Questions","date":"2017-03-31","rows_on_this_dataset":1,"code_links":10,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":1,"samples_ran":1,"samples_unverified":0,"pointer_only_for_licence":1,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}}],"syntology_totals":{"read_at":"2026-09-24T18:15:14+00:00","papers_with_samples":25,"samples_harvested":387,"samples_ran":192,"samples_unverified":195,"pointer_only_for_licence":66,"papers_with_no_sample_that_ran":3,"note":"the per-paper counts above, summed; not a rate"},"papers_note":"The archive never published its papers-using-dataset list; these are papers with a leaderboard row on this dataset's benchmarks."}