{"url":"/dataset/theoremqa","name":"TheoremQA","full_name":"TheoremQA","description_markdown":"We propose the first question-answering dataset driven by STEM theorems. We annotated 800 QA pairs covering 350+ theorems spanning across Math, EE&CS, Physics and Finance. The dataset is collected by human experts with very high quality. We provide the dataset as a new benchmark to test the limit of large language models to apply theorems to solve challenging university-level questions. We provide a pipeline in the following to prompt LLMs and evaluate their outputs with WolframAlpha.","description_withheld":null,"homepage":"https://github.com/wenhuchen/TheoremQA","introduced_date":"2023-05-21","introduced_date_note":null,"introduced_by":{"paper":"/paper/theoremqa-a-theorem-driven-question-answering","title":"TheoremQA: A Theorem-driven Question Answering dataset","first_author":"Wenhu Chen","url":null},"license":{"name":"MIT","url":"https://raw.githubusercontent.com/wenhuchen/TheoremQA/main/LICENSE"},"modalities":[{"name":"Images","url":"/datasets/modality/images"},{"name":"Texts","url":"/datasets/modality/texts"}],"tasks":[{"name":"Question Answering","url":"/task/question-answering","datasets_with_task":"/datasets/task/question-answering"},{"name":"Mathematical Reasoning","url":"/task/mathematical-reasoning","datasets_with_task":"/datasets/task/mathematical-reasoning"},{"name":"Natural Questions","url":"/task/natural-questions","datasets_with_task":"/datasets/task/natural-questions"},{"name":"Mathematical Problem-Solving","url":"/task/mathematical-problem-solving","datasets_with_task":"/datasets/task/mathematical-problem-solving"}],"languages":[{"name":"English","url":"/datasets/language/english"}],"variants":["TheoremQA"],"data_loaders":[],"num_papers_in_archive":40,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28"},"benchmarks":[{"leaderboard":"/sota/natural-questions-on-theoremqa","task":"Natural Questions","dataset_variant":"TheoremQA","rows":19,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"GPT-4 (PoT)","paper":"/paper/theoremqa-a-theorem-driven-question-answering","metrics":{"Accuracy":"52.4"},"code_links":[{"title":"wenhuchen/theoremqa","url":"https://github.com/wenhuchen/theoremqa"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"}],"papers_with_a_benchmark_row":[{"paper":"/paper/dart-math-difficulty-aware-rejection-tuning-1","title":"DART-Math: Difficulty-Aware Rejection Tuning for Mathematical Problem-Solving","date":"2024-06-18","rows_on_this_dataset":8,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":12,"samples_ran":10,"samples_unverified":2,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/theoremqa-a-theorem-driven-question-answering","title":"TheoremQA: A Theorem-driven Question Answering dataset","date":"2023-05-21","rows_on_this_dataset":11,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":10,"samples_ran":2,"samples_unverified":8,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}}],"syntology_totals":{"read_at":"2026-09-24T18:15:14+00:00","papers_with_samples":2,"samples_harvested":22,"samples_ran":12,"samples_unverified":10,"pointer_only_for_licence":0,"papers_with_no_sample_that_ran":0,"note":"the per-paper counts above, summed; not a rate"},"papers_note":"The archive never published its papers-using-dataset list; these are papers with a leaderboard row on this dataset's benchmarks."}