{"url":"/dataset/pubmedqa","name":"PubMedQA","full_name":"PubMedQA","description_markdown":"The task of PubMedQA is to answer research questions with yes/no/maybe (e.g.: Do preoperative statins reduce atrial fibrillation after coronary artery bypass grafting?) using the corresponding abstracts.\r\n\r\nPubMedQA has 1k expert labeled, 61.2k unlabeled and 211.3k artificially generated QA instances.\r\n\r\nSource: [PubMedQA](https://pubmedqa.github.io/)\r\nImage Source: [https://arxiv.org/pdf/1909.06146v1.pdf](https://arxiv.org/pdf/1909.06146v1.pdf)","description_withheld":null,"homepage":"https://pubmedqa.github.io/","introduced_date":"2019-09-13","introduced_date_note":null,"introduced_by":{"paper":"/paper/pubmedqa-a-dataset-for-biomedical-research","title":"PubMedQA: A Dataset for Biomedical Research Question Answering","first_author":"Qiao Jin","url":null},"license":{"name":"Custom","url":"https://pubmedqa.github.io/"},"modalities":[{"name":"Texts","url":"/datasets/modality/texts"}],"tasks":[{"name":"Question Answering","url":"/task/question-answering","datasets_with_task":"/datasets/task/question-answering"},{"name":"Named Entity Recognition (NER)","url":"/task/named-entity-recognition-ner","datasets_with_task":"/datasets/task/named-entity-recognition-ner"},{"name":"Few-Shot Learning","url":"/task/few-shot-learning","datasets_with_task":"/datasets/task/few-shot-learning"},{"name":"Language Modelling","url":"/task/language-modelling","datasets_with_task":"/datasets/task/language-modelling"},{"name":"Retrieval","url":"/task/retrieval","datasets_with_task":"/datasets/task/retrieval"}],"languages":[],"variants":["PubMedQA"],"data_loaders":[{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/qiaojin/PubMedQA","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/qiaojin/pubmed_qa","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/highnote/pubmed_qa","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/pubmed_qa","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/ibm-self-serve-assets/metagen-blended-rag","url":"https://huggingface.co/datasets/Shivam6693/PubMedQA-MetaGenBlendedRAG/blob/main/README.md","frameworks":[]}],"num_papers_in_archive":276,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28"},"benchmarks":[{"leaderboard":"/sota/question-answering-on-pubmedqa","task":"Question Answering","dataset_variant":"PubMedQA","rows":30,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Meditron-70B (CoT + SC)","paper":"/paper/meditron-70b-scaling-medical-pretraining-for","metrics":{"Accuracy":"81.6"},"code_links":[{"title":"epfllm/meditron","url":"https://github.com/epfllm/meditron"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/few-shot-learning-on-pubmedqa","task":"Few-Shot Learning","dataset_variant":"PubMedQA","rows":2,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"MetaGen Blended RAG (zero-shot)","paper":"/paper/metagen-blended-rag-higher-accuracy-for","metrics":{"Accuracy":"77.9"},"code_links":[{"title":"ibm-self-serve-assets/metagen-blended-rag","url":"https://github.com/ibm-self-serve-assets/metagen-blended-rag"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/retrieval-on-pubmedqa","task":"Retrieval","dataset_variant":"PubMedQA","rows":1,"metrics":["Accuracy (Top-1)"],"first_row_in_archive_order":{"model":"MetaGen Blended RAG","paper":"/paper/metagen-blended-rag-higher-accuracy-for","metrics":{"Accuracy (Top-1)":"82.1"},"code_links":[{"title":"ibm-self-serve-assets/metagen-blended-rag","url":"https://github.com/ibm-self-serve-assets/metagen-blended-rag"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"}],"papers_with_a_benchmark_row":[{"paper":"/paper/metagen-blended-rag-higher-accuracy-for","title":"MetaGen Blended RAG: Higher Accuracy for Domain-Specific Q&A Without Fine-Tuning","date":"2025-05-23","rows_on_this_dataset":3,"code_links":1,"syntology":null},{"paper":"/paper/rankrag-unifying-context-ranking-with","title":"RankRAG: Unifying Context Ranking with Retrieval-Augmented Generation in LLMs","date":"2024-07-02","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/evaluation-of-large-language-model","title":"Evaluation of large language model performance on the Biomedical Language Understanding and Reasoning Benchmark","date":"2024-05-17","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/the-claude-3-model-family-opus-sonnet-haiku","title":"The Claude 3 Model Family: Opus, Sonnet, Haiku","date":"2024-03-04","rows_on_this_dataset":2,"code_links":0,"syntology":null},{"paper":"/paper/mediswift-efficient-sparse-pre-trained","title":"MediSwift: Efficient Sparse Pre-trained Biomedical Language Models","date":"2024-03-01","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/meditron-70b-scaling-medical-pretraining-for","title":"MEDITRON-70B: Scaling Medical Pretraining for Large Language Models","date":"2023-11-27","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":14,"samples_ran":9,"samples_unverified":5,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/biomedgpt-open-multimodal-generative-pre","title":"BioMedGPT: Open Multimodal Generative Pre-trained Transformer for BioMedicine","date":"2023-08-18","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/the-cot-collection-improving-zero-shot-and","title":"The CoT Collection: Improving Zero-shot and Few-shot Learning of Language Models via Chain-of-Thought Fine-Tuning","date":"2023-05-23","rows_on_this_dataset":2,"code_links":2,"syntology":null},{"paper":"/paper/towards-expert-level-medical-question","title":"Towards Expert-Level Medical Question Answering with Large Language Models","date":"2023-05-16","rows_on_this_dataset":3,"code_links":1,"syntology":null},{"paper":"/paper/large-language-models-encode-clinical","title":"Large Language Models Encode Clinical Knowledge","date":"2022-12-26","rows_on_this_dataset":7,"code_links":1,"syntology":null},{"paper":"/paper/galactica-a-large-language-model-for-science-1","title":"Galactica: A Large Language Model for Science","date":"2022-11-16","rows_on_this_dataset":3,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":2,"samples_ran":0,"samples_unverified":2,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/biogpt-generative-pre-trained-transformer-for","title":"BioGPT: Generative Pre-trained Transformer for Biomedical Text Generation and Mining","date":"2022-10-19","rows_on_this_dataset":2,"code_links":4,"syntology":null},{"paper":"/paper/can-large-language-models-reason-about","title":"Can large language models reason about medical questions?","date":"2022-07-17","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":10,"samples_ran":0,"samples_unverified":10,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/linkbert-pretraining-language-models-with","title":"LinkBERT: Pretraining Language Models with Document Links","date":"2022-03-29","rows_on_this_dataset":2,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":14,"samples_ran":0,"samples_unverified":14,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/bioelectra-pretrained-biomedical-text-encoder","title":"BioELECTRA:Pretrained Biomedical text Encoder using Discriminators","date":"2021-06-11","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/domain-specific-language-model-pretraining","title":"Domain-Specific Language Model Pretraining for Biomedical Natural Language Processing","date":"2020-07-31","rows_on_this_dataset":1,"code_links":2,"syntology":null},{"paper":"/paper/pubmedqa-a-dataset-for-biomedical-research","title":"PubMedQA: A Dataset for Biomedical Research Question Answering","date":"2019-09-13","rows_on_this_dataset":1,"code_links":5,"syntology":null}],"syntology_totals":{"read_at":"2026-09-24T18:15:14+00:00","papers_with_samples":4,"samples_harvested":40,"samples_ran":9,"samples_unverified":31,"pointer_only_for_licence":0,"papers_with_no_sample_that_ran":3,"note":"the per-paper counts above, summed; not a rate"},"papers_note":"The archive never published its papers-using-dataset list; these are papers with a leaderboard row on this dataset's benchmarks."}