{"url":"/dataset/iconqa","name":"IconQA","full_name":"Icon Question Answering","description_markdown":"Current visual question answering (VQA) tasks mainly consider answering human-annotated questions for natural images in the daily-life context. **Icon question answering** (**IconQA**) is a benchmark which aims to highlight the importance of abstract diagram understanding and comprehensive cognitive reasoning in real-world diagram word problems. For this benchmark, a large-scale IconQA dataset is built that consists of three sub-tasks: multi-image-choice, multi-text-choice, and filling-in-the-blank. Compared to existing [VQA benchmarks](https://paperswithcode.com/datasets?task=visual-question-answering), IconQA requires not only perception skills like object recognition and text understanding, but also diverse cognitive reasoning skills, such as geometric reasoning, commonsense reasoning, and arithmetic reasoning.\r\n\r\nDescription from: [IconQA](https://iconqa.github.io/)","description_withheld":null,"homepage":"https://iconqa.github.io/","introduced_date":"2021-10-25","introduced_date_note":null,"introduced_by":{"paper":"/paper/iconqa-a-new-benchmark-for-abstract-diagram","title":"IconQA: A New Benchmark for Abstract Diagram Understanding and Visual Language Reasoning","first_author":"Pan Lu","url":null},"license":{"name":"CC BY-NC-SA 4.0","url":"https://creativecommons.org/licenses/by-nc-sa/4.0/"},"modalities":[{"name":"Images","url":"/datasets/modality/images"},{"name":"Texts","url":"/datasets/modality/texts"}],"tasks":[{"name":"Visual Question Answering (VQA)","url":"/task/visual-question-answering","datasets_with_task":"/datasets/task/visual-question-answering"},{"name":"Common Sense Reasoning","url":"/task/common-sense-reasoning","datasets_with_task":"/datasets/task/common-sense-reasoning"},{"name":"Mathematical Reasoning","url":"/task/mathematical-reasoning","datasets_with_task":"/datasets/task/mathematical-reasoning"},{"name":"Visual Reasoning","url":"/task/visual-reasoning","datasets_with_task":"/datasets/task/visual-reasoning"},{"name":"Mathematical Question Answering","url":"/task/mathematical-question-answering","datasets_with_task":"/datasets/task/mathematical-question-answering"},{"name":"Math Word Problem Solving","url":"/task/math-word-problem-solving","datasets_with_task":"/datasets/task/math-word-problem-solving"},{"name":"Visual Commonsense Reasoning","url":"/task/visual-commonsense-reasoning","datasets_with_task":"/datasets/task/visual-commonsense-reasoning"}],"languages":[{"name":"English","url":"/datasets/language/english"}],"variants":["IconQA"],"data_loaders":[{"repo":"https://github.com/lupantech/iconqa","url":"https://github.com/lupantech/iconqa","frameworks":[]}],"num_papers_in_archive":42,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28"},"benchmarks":[{"leaderboard":"/sota/visual-question-answering-on-iconqa","task":"Visual Question Answering (VQA)","dataset_variant":"IconQA","rows":12,"metrics":["Sub-tasks (Img.)","Sub-tasks (Txt.)","Sub-tasks (Blank)","Reasoning (Geo.)","Reasoning (Cou.)","Reasoning (Com.)","Reasoning (Spa.)","Reasoning (Sce.)","Reasoning (Pat.)","Reasoning (Tim.)","Reasoning (Fra.)","Reasoning (Est.)","Reasoning (Alg.)","Reasoning (Mea.)","Reasoning (Sen.)","Reasoning (Pro.)"],"first_row_in_archive_order":{"model":"Patch-TRM","paper":"/paper/iconqa-a-new-benchmark-for-abstract-diagram","metrics":{"Reasoning (Alg.)":"56.73","Reasoning (Com.)":"87.00","Reasoning (Cou.)":"77.81","Reasoning (Est.)":"98.24","Reasoning (Fra.)":"82.13","Reasoning (Geo.)":"81.87","Reasoning (Mea.)":"97.98","Reasoning (Pat.)":"68.75","Reasoning (Pro.)":"95.73","Reasoning (Sce.)":"62.39","Reasoning (Sen.)":"92.49","Reasoning (Spa.)":"55.62","Reasoning (Tim.)":"77.98","Sub-tasks (Blank)":"83.62","Sub-tasks (Img.)":"82.66","Sub-tasks (Txt.)":"75.19"},"code_links":[{"title":"lupantech/iconqa","url":"https://github.com/lupantech/iconqa"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"}],"papers_with_a_benchmark_row":[{"paper":"/paper/iconqa-a-new-benchmark-for-abstract-diagram","title":"IconQA: A New Benchmark for Abstract Diagram Understanding and Visual Language Reasoning","date":"2021-10-25","rows_on_this_dataset":12,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":10,"samples_ran":8,"samples_unverified":2,"pointer_only_for_licence":10,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}}],"syntology_totals":{"read_at":"2026-09-24T18:15:14+00:00","papers_with_samples":1,"samples_harvested":10,"samples_ran":8,"samples_unverified":2,"pointer_only_for_licence":10,"papers_with_no_sample_that_ran":0,"note":"the per-paper counts above, summed; not a rate"},"papers_note":"The archive never published its papers-using-dataset list; these are papers with a leaderboard row on this dataset's benchmarks."}