{"url":"/dataset/illusionvqa","name":"IllusionVQA","full_name":null,"description_markdown":"IllusionVQA is a Visual Question Answering (VQA) dataset with two sub-tasks. The first task tests comprehension on 435 instances in 12 optical illusion categories. Each instance consists of an image with an optical illusion, a question, and 3 to 6 options, one of which is the correct answer. We refer to this task as Logo IllusionVQA-Comprehension. The second task tests how well VLMs can differentiate geometrically impossible objects from ordinary objects when two objects are presented side by side. The task consists of 1000 instances following a similar format to the first task. We refer to this task as Logo IllusionVQA-Soft-Localization.","description_withheld":null,"homepage":"https://illusionvqa.github.io/","introduced_date":"2024-03-23","introduced_date_note":null,"introduced_by":{"paper":"/paper/illusionvqa-a-challenging-optical-illusion","title":"IllusionVQA: A Challenging Optical Illusion Dataset for Vision Language Models","first_author":"HAZ Sameen Shahgir","url":null},"license":{"name":"CC BY-NC-SA 4.0","url":"https://creativecommons.org/licenses/by-nc-sa/4.0/"},"modalities":[{"name":"Images","url":"/datasets/modality/images"},{"name":"Texts","url":"/datasets/modality/texts"}],"tasks":[{"name":"Visual Question Answering (VQA)","url":"/task/visual-question-answering","datasets_with_task":"/datasets/task/visual-question-answering"},{"name":"Object Localization","url":"/task/object-localization","datasets_with_task":"/datasets/task/object-localization"}],"languages":[{"name":"English","url":"/datasets/language/english"}],"variants":["IllusionVQA"],"data_loaders":[],"num_papers_in_archive":3,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28"},"benchmarks":[{"leaderboard":"/sota/object-localization-on-illusionvqa","task":"Object Localization","dataset_variant":"IllusionVQA","rows":9,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"GPT4-Vision 4-shot+CoT","paper":"/paper/illusionvqa-a-challenging-optical-illusion","metrics":{"Accuracy":"49.7"},"code_links":[{"title":"csebuetnlp/illusionvqa","url":"https://github.com/csebuetnlp/illusionvqa"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/visual-question-answering-vqa-on-illusionvqa","task":"Visual Question Answering (VQA)","dataset_variant":"IllusionVQA","rows":7,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"GPT4-Vision 4-shot","paper":"/paper/illusionvqa-a-challenging-optical-illusion","metrics":{"Accuracy":"62.99"},"code_links":[{"title":"csebuetnlp/illusionvqa","url":"https://github.com/csebuetnlp/illusionvqa"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"}],"papers_with_a_benchmark_row":[{"paper":"/paper/illusionvqa-a-challenging-optical-illusion","title":"IllusionVQA: A Challenging Optical Illusion Dataset for Vision Language Models","date":"2024-03-23","rows_on_this_dataset":16,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":5,"samples_ran":5,"samples_unverified":0,"pointer_only_for_licence":5,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}}],"syntology_totals":{"read_at":"2026-09-24T18:15:14+00:00","papers_with_samples":1,"samples_harvested":5,"samples_ran":5,"samples_unverified":0,"pointer_only_for_licence":5,"papers_with_no_sample_that_ran":0,"note":"the per-paper counts above, summed; not a rate"},"papers_note":"The archive never published its papers-using-dataset list; these are papers with a leaderboard row on this dataset's benchmarks."}