{"url":"/dataset/mmlu","name":"MML","full_name":"Massive Multitask Language Understanding","description_markdown":"**MMLU** (**Massive Multitask Language Understanding**) is a new benchmark designed to measure knowledge acquired during pretraining by evaluating models exclusively in zero-shot and few-shot settings. This makes the benchmark more challenging and more similar to how we evaluate humans. The benchmark covers 57 subjects across STEM, the humanities, the social sciences, and more. It ranges in difficulty from an elementary level to an advanced professional level, and it tests both world knowledge and problem solving ability. Subjects range from traditional areas, such as mathematics and history, to more specialized areas like law and ethics. The granularity and breadth of the subjects makes the benchmark ideal for identifying a model’s blind spots.\r\n\r\nImage source: [https://arxiv.org/pdf/2009.03300v3.pdf](https://arxiv.org/pdf/2009.03300v3.pdf)","description_withheld":null,"homepage":"https://github.com/hendrycks/test","introduced_date":"2020-09-07","introduced_date_note":null,"introduced_by":{"paper":"/paper/measuring-massive-multitask-language","title":"Measuring Massive Multitask Language Understanding","first_author":"Dan Hendrycks","url":null},"license":{"name":"Custom","url":"https://github.com/hendrycks/test#citation"},"modalities":[{"name":"Texts","url":"/datasets/modality/texts"}],"tasks":[{"name":"Question Answering","url":"/task/question-answering","datasets_with_task":"/datasets/task/question-answering"},{"name":"Text Generation","url":"/task/text-generation","datasets_with_task":"/datasets/task/text-generation"},{"name":"Multiple Choice Question Answering (MCQA)","url":"/task/multiple-choice-qa","datasets_with_task":"/datasets/task/multiple-choice-qa"},{"name":"Natural Language Understanding","url":"/task/natural-language-understanding","datasets_with_task":"/datasets/task/natural-language-understanding"},{"name":"Multi-task Language Understanding","url":"/task/multi-task-language-understanding","datasets_with_task":"/datasets/task/multi-task-language-understanding"},{"name":"Single Choice Question","url":"/task/single-choice-question","datasets_with_task":"/datasets/task/single-choice-question"}],"languages":[{"name":"English","url":"/datasets/language/english"}],"variants":["mmlu","MMLU TR v0.2","MMLU TR","MMLU (5-Shot)","MMLU (College Medicine)","MMLU (Professional medicine)","MMLU (Anatomy)","MMLU (Clinical Knowledge)","MMLU (Medical Genetics)","MMLU (Mathematics)","MMLU (Machine Learning)","MMLU (High School Statistics)","MMLU (High School Physics)","MMLU (High School Mathematics)","MMLU (High School Computer Science)","MMLU (High School Chemistry)","MMLU (High School Biology)","MMLU (Formal Logic)","MMLU (Elementary Mathematics)","MMLU (Electrical Engineer)","MMLU (Econometrics)","MMLU (College Physics)","MMLU (College Mathematics)","MMLU (College Computer Science)","MMLU (College Chemistry)","MMLU (College Biology)","MMLU (Astronomy)","MMLU (Abstract Algebra)","MML"],"data_loaders":[{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/hendrycks_test","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/cais/mmlu","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/lighteval/mmlu","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/Stevross/mmlu","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/swap-uniba/mmlu_ita","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/Weyaxi/mmlu","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/justinphan3110/mmlu-test","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/justinphan3110/mmlu-test-2","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/neurotechnology/lt_mmlu","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/flunardelli/mmlu","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/rubricreward/llm-metric-mmlu","frameworks":["tf","pytorch","jax"]}],"num_papers_in_archive":1922,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28"},"benchmarks":[{"leaderboard":"/sota/multi-task-language-understanding-on-mmlu","task":"Multi-task Language Understanding","dataset_variant":"MML","rows":44,"metrics":["Average (%)"],"first_row_in_archive_order":{"model":"GPT-4 o1(300b)","paper":"/paper/sieve-general-purpose-data-filtering-system","metrics":{"Average (%)":"87"},"code_links":[]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-11","task":"Multiple Choice Question Answering (MCQA)","dataset_variant":"MMLU (College Biology)","rows":8,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Med-PaLM 2 (ER)","paper":"/paper/towards-expert-level-medical-question","metrics":{"Accuracy":"95.8"},"code_links":[{"title":"m42-health/med42","url":"https://github.com/m42-health/med42"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-8","task":"Multiple Choice Question Answering (MCQA)","dataset_variant":"MMLU (Medical Genetics)","rows":8,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Med-PaLM 2 (ER)","paper":"/paper/towards-expert-level-medical-question","metrics":{"Accuracy":"92"},"code_links":[{"title":"m42-health/med42","url":"https://github.com/m42-health/med42"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-25","task":"Multiple Choice Question Answering (MCQA)","dataset_variant":"MMLU (Professional medicine)","rows":6,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Med-PaLM 2 (5-shot)","paper":"/paper/towards-expert-level-medical-question","metrics":{"Accuracy":"95.2"},"code_links":[{"title":"m42-health/med42","url":"https://github.com/m42-health/med42"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-10","task":"Multiple Choice Question Answering (MCQA)","dataset_variant":"MMLU (Elementary Mathematics)","rows":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla (few-shot, k=5)","paper":"/paper/galactica-a-large-language-model-for-science-1","metrics":{"Accuracy":"41.5"},"code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-12","task":"Multiple Choice Question Answering (MCQA)","dataset_variant":"MMLU (High School Biology)","rows":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla (few-shot, k=5)","paper":"/paper/galactica-a-large-language-model-for-science-1","metrics":{"Accuracy":"80.3"},"code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-13","task":"Multiple Choice Question Answering (MCQA)","dataset_variant":"MMLU (College Chemistry)","rows":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla (few-shot, k=5)","paper":"/paper/galactica-a-large-language-model-for-science-1","metrics":{"Accuracy":"51"},"code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-16","task":"Multiple Choice Question Answering (MCQA)","dataset_variant":"MMLU (High School Mathematics)","rows":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"GAL 120B (zero-shot)","paper":"/paper/galactica-a-large-language-model-for-science-1","metrics":{"Accuracy":"32.6"},"code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-17","task":"Multiple Choice Question Answering (MCQA)","dataset_variant":"MMLU (Electrical Engineer)","rows":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"GAL 120B (zero-shot)","paper":"/paper/galactica-a-large-language-model-for-science-1","metrics":{"Accuracy":"62.8"},"code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-18","task":"Multiple Choice Question Answering (MCQA)","dataset_variant":"MMLU (College Physics)","rows":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla (few-shot, k=5)","paper":"/paper/galactica-a-large-language-model-for-science-1","metrics":{"Accuracy":"46.1"},"code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-2","task":"Multiple Choice Question Answering (MCQA)","dataset_variant":"MMLU (Formal Logic)","rows":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Gopher (few-shot, k=5)","paper":"/paper/galactica-a-large-language-model-for-science-1","metrics":{"Accuracy":"35.7"},"code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-20","task":"Multiple Choice Question Answering (MCQA)","dataset_variant":"MMLU (High School Statistics)","rows":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla (few-shot, k=5)","paper":"/paper/galactica-a-large-language-model-for-science-1","metrics":{"Accuracy":"58.8"},"code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-3","task":"Multiple Choice Question Answering (MCQA)","dataset_variant":"MMLU (Abstract Algebra)","rows":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"GAL 30B (zero-shot)","paper":"/paper/galactica-a-large-language-model-for-science-1","metrics":{"Accuracy":"33.3"},"code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-4","task":"Multiple Choice Question Answering (MCQA)","dataset_variant":"MMLU (Econometrics)","rows":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Gopher (few-shot, k=5)","paper":"/paper/galactica-a-large-language-model-for-science-1","metrics":{"Accuracy":"43"},"code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-5","task":"Multiple Choice Question Answering (MCQA)","dataset_variant":"MMLU (High School Computer Science)","rows":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"GAL 120B (zero-shot)","paper":"/paper/galactica-a-large-language-model-for-science-1","metrics":{"Accuracy":"70"},"code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-7","task":"Multiple Choice Question Answering (MCQA)","dataset_variant":"MMLU (College Mathematics)","rows":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"GAL 120B (zero-shot)","paper":"/paper/galactica-a-large-language-model-for-science-1","metrics":{"Accuracy":"43"},"code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-9","task":"Multiple Choice Question Answering (MCQA)","dataset_variant":"MMLU (Astronomy)","rows":5,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla (few-shot, k=5)","paper":"/paper/galactica-a-large-language-model-for-science-1","metrics":{"Accuracy":"73.0"},"code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-14","task":"Multiple Choice Question Answering (MCQA)","dataset_variant":"MMLU (High School Chemistry)","rows":4,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla (few-shot, k=5)","paper":"/paper/galactica-a-large-language-model-for-science-1","metrics":{"Accuracy":"58.1"},"code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-15","task":"Multiple Choice Question Answering (MCQA)","dataset_variant":"MMLU (College Computer Science)","rows":4,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla (few-shot, k=5)","paper":"/paper/galactica-a-large-language-model-for-science-1","metrics":{"Accuracy":"51.0"},"code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-19","task":"Multiple Choice Question Answering (MCQA)","dataset_variant":"MMLU (High School Physics)","rows":4,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla (few-shot, k=5)","paper":"/paper/galactica-a-large-language-model-for-science-1","metrics":{"Accuracy":"36.4"},"code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-6","task":"Multiple Choice Question Answering (MCQA)","dataset_variant":"MMLU (Machine Learning)","rows":4,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Chinchilla (few-shot, k=5)","paper":"/paper/galactica-a-large-language-model-for-science-1","metrics":{"Accuracy":"41.1"},"code_links":[{"title":"paperswithcode/galai","url":"https://github.com/paperswithcode/galai"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-23","task":"Multiple Choice Question Answering (MCQA)","dataset_variant":"MMLU (Clinical Knowledge)","rows":3,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Med-PaLM 2 (ER)","paper":"/paper/towards-expert-level-medical-question","metrics":{"Accuracy":"88.7"},"code_links":[{"title":"m42-health/med42","url":"https://github.com/m42-health/med42"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-24","task":"Multiple Choice Question Answering (MCQA)","dataset_variant":"MMLU (Anatomy)","rows":3,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Med-PaLM 2 (ER)","paper":"/paper/towards-expert-level-medical-question","metrics":{"Accuracy":"84.4"},"code_links":[{"title":"m42-health/med42","url":"https://github.com/m42-health/med42"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multiple-choice-question-answering-mcqa-on-26","task":"Multiple Choice Question Answering (MCQA)","dataset_variant":"MMLU (College Medicine)","rows":3,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"Med-PaLM (ER)","paper":"/paper/towards-expert-level-medical-question","metrics":{"Accuracy":"83.2"},"code_links":[{"title":"m42-health/med42","url":"https://github.com/m42-health/med42"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multi-task-language-understanding-on-mmlu-5-1","task":"Multi-task Language Understanding","dataset_variant":"MMLU (5-Shot)","rows":1,"metrics":["MMLU (5-shot)"],"first_row_in_archive_order":{"model":"Sakalti/ultiima-78B","paper":"/paper/merge-fast-private-text-generation","metrics":{"MMLU (5-shot)":"89.2"},"code_links":[{"title":"liangzid/MERGE","url":"https://github.com/liangzid/MERGE"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/question-answering-on-mmlu","task":"Question Answering","dataset_variant":"MML","rows":1,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"qwen-LLM 7B","paper":"/paper/shakti-a-2-5-billion-parameter-small-language","metrics":{"Accuracy":"71.8"},"code_links":[]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/text-generation-on-mmlu-5-shot","task":"Text Generation","dataset_variant":"MMLU (5-Shot)","rows":0,"metrics":["accuracy"],"first_row_in_archive_order":null,"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/text-generation-on-mmlu-tr","task":"Text Generation","dataset_variant":"MMLU TR","rows":0,"metrics":["accuracy"],"first_row_in_archive_order":null,"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/text-generation-on-mmlu-tr-v0-2","task":"Text Generation","dataset_variant":"MMLU TR v0.2","rows":0,"metrics":["accuracy"],"first_row_in_archive_order":null,"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"}],"papers_with_a_benchmark_row":[{"paper":"/paper/llama-3-meets-moe-efficient-upcycling","title":"Llama 3 Meets MoE: Efficient Upcycling","date":"2024-12-13","rows_on_this_dataset":2,"code_links":1,"syntology":null},{"paper":"/paper/shakti-a-2-5-billion-parameter-small-language","title":"SHAKTI: A 2.5 Billion Parameter Small Language Model Optimized for Edge AI and Low-Resource Environments","date":"2024-10-15","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/sieve-general-purpose-data-filtering-system","title":"GPT-4o as the Gold Standard: A Scalable and General Purpose Approach to Filter Language Model Pretraining Data","date":"2024-10-03","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/the-llama-3-herd-of-models","title":"The Llama 3 Herd of Models","date":"2024-07-31","rows_on_this_dataset":2,"code_links":5,"syntology":{"read_at":"2026-09-25T09:33:49+00:00","samples_harvested":9,"samples_ran":9,"samples_unverified":0,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/branch-train-mix-mixing-expert-llms-into-a","title":"Branch-Train-MiX: Mixing Expert LLMs into a Mixture-of-Experts LLM","date":"2024-03-12","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/the-claude-3-model-family-opus-sonnet-haiku","title":"The Claude 3 Model Family: Opus, Sonnet, Haiku","date":"2024-03-04","rows_on_this_dataset":2,"code_links":0,"syntology":null},{"paper":"/paper/mixtral-of-experts","title":"Mixtral of Experts","date":"2024-01-08","rows_on_this_dataset":2,"code_links":6,"syntology":{"read_at":"2026-09-25T09:33:49+00:00","samples_harvested":5,"samples_ran":5,"samples_unverified":0,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/the-falcon-series-of-open-language-models","title":"The Falcon Series of Open Language Models","date":"2023-11-28","rows_on_this_dataset":2,"code_links":0,"syntology":null},{"paper":"/paper/infoentropy-loss-to-mitigate-bias-of-learning","title":"MiLe Loss: a New Loss for Mitigating the Bias of Learning Difficulties in Generative Language Models","date":"2023-10-30","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/mistral-7b","title":"Mistral 7B","date":"2023-10-10","rows_on_this_dataset":1,"code_links":6,"syntology":{"read_at":"2026-09-25T09:33:49+00:00","samples_harvested":11,"samples_ran":10,"samples_unverified":1,"pointer_only_for_licence":1,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/textbooks-are-all-you-need-ii-phi-1-5","title":"Textbooks Are All You Need II: phi-1.5 technical report","date":"2023-09-11","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/biomedgpt-open-multimodal-generative-pre","title":"BioMedGPT: Open Multimodal Generative Pre-trained Transformer for BioMedicine","date":"2023-08-18","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/llama-2-open-foundation-and-fine-tuned-chat","title":"Llama 2: Open Foundation and Fine-Tuned Chat Models","date":"2023-07-18","rows_on_this_dataset":5,"code_links":19,"syntology":{"read_at":"2026-09-25T09:33:49+00:00","samples_harvested":52,"samples_ran":31,"samples_unverified":21,"pointer_only_for_licence":16,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/merge-fast-private-text-generation","title":"MERGE: Fast Private Text Generation","date":"2023-05-25","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/towards-expert-level-medical-question","title":"Towards Expert-Level Medical Question Answering with Large Language Models","date":"2023-05-16","rows_on_this_dataset":18,"code_links":1,"syntology":null},{"paper":"/paper/bloomberggpt-a-large-language-model-for","title":"BloombergGPT: A Large Language Model for Finance","date":"2023-03-30","rows_on_this_dataset":3,"code_links":2,"syntology":null},{"paper":"/paper/gpt-4-technical-report-1","title":"GPT-4 Technical Report","date":"2023-03-15","rows_on_this_dataset":1,"code_links":11,"syntology":{"read_at":"2026-09-25T09:33:49+00:00","samples_harvested":5,"samples_ran":5,"samples_unverified":0,"pointer_only_for_licence":1,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/llama-open-and-efficient-foundation-language-1","title":"LLaMA: Open and Efficient Foundation Language Models","date":"2023-02-27","rows_on_this_dataset":3,"code_links":57,"syntology":{"read_at":"2026-09-25T09:33:49+00:00","samples_harvested":58,"samples_ran":37,"samples_unverified":21,"pointer_only_for_licence":4,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/galactica-a-large-language-model-for-science-1","title":"Galactica: A Large Language Model for Science","date":"2022-11-16","rows_on_this_dataset":92,"code_links":1,"syntology":{"read_at":"2026-09-25T09:33:49+00:00","samples_harvested":2,"samples_ran":2,"samples_unverified":0,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/scaling-instruction-finetuned-language-models","title":"Scaling Instruction-Finetuned Language Models","date":"2022-10-20","rows_on_this_dataset":9,"code_links":9,"syntology":{"read_at":"2026-09-25T09:33:49+00:00","samples_harvested":17,"samples_ran":8,"samples_unverified":9,"pointer_only_for_licence":2,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/glm-130b-an-open-bilingual-pre-trained-model","title":"GLM-130B: An Open Bilingual Pre-trained Model","date":"2022-10-05","rows_on_this_dataset":1,"code_links":9,"syntology":{"read_at":"2026-09-25T09:33:49+00:00","samples_harvested":21,"samples_ran":13,"samples_unverified":8,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/few-shot-learning-with-retrieval-augmented","title":"Atlas: Few-shot Learning with Retrieval Augmented Language Models","date":"2022-08-05","rows_on_this_dataset":1,"code_links":2,"syntology":{"read_at":"2026-09-25T09:33:49+00:00","samples_harvested":2,"samples_ran":2,"samples_unverified":0,"pointer_only_for_licence":2,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/unifying-language-learning-paradigms","title":"UL2: Unifying Language Learning Paradigms","date":"2022-05-10","rows_on_this_dataset":1,"code_links":2,"syntology":{"read_at":"2026-09-25T09:33:49+00:00","samples_harvested":16,"samples_ran":15,"samples_unverified":1,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/gpt-neox-20b-an-open-source-autoregressive-1","title":"GPT-NeoX-20B: An Open-Source Autoregressive Language Model","date":"2022-04-14","rows_on_this_dataset":1,"code_links":11,"syntology":{"read_at":"2026-09-25T09:33:49+00:00","samples_harvested":3,"samples_ran":2,"samples_unverified":1,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/training-compute-optimal-large-language","title":"Training Compute-Optimal Large Language Models","date":"2022-03-29","rows_on_this_dataset":1,"code_links":2,"syntology":{"read_at":"2026-09-25T09:33:49+00:00","samples_harvested":11,"samples_ran":8,"samples_unverified":3,"pointer_only_for_licence":4,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}}],"syntology_totals":{"read_at":"2026-09-25T09:33:49+00:00","papers_with_samples":13,"samples_harvested":212,"samples_ran":147,"samples_unverified":65,"pointer_only_for_licence":30,"papers_with_no_sample_that_ran":0,"note":"the per-paper counts above, summed; not a rate"},"papers_note":"The archive never published its papers-using-dataset list; these are papers with a leaderboard row on this dataset's benchmarks."}