{"url":"/dataset/vqa-ce","name":"VQA-CE","full_name":"VQA Counterexamples","description_markdown":"This dataset provides a new split of VQA v2 (similarly to VQA-CP v2), which is built of questions that are hard to answer for biased models.\r\n\r\nThis dataset is designed to penalize biases, and encourage the learning of models that generalize well.","description_withheld":null,"homepage":"https://arxiv.org/abs/2104.03149","introduced_date":"2021-04-07","introduced_date_note":null,"introduced_by":{"paper":"/paper/beyond-question-based-biases-assessing","title":"Beyond Question-Based Biases: Assessing Multimodal Shortcut Learning in Visual Question Answering","first_author":"Corentin Dancette","url":null},"license":null,"modalities":[{"name":"Images","url":"/datasets/modality/images"},{"name":"Texts","url":"/datasets/modality/texts"}],"tasks":[{"name":"Visual Question Answering (VQA)","url":"/task/visual-question-answering","datasets_with_task":"/datasets/task/visual-question-answering"}],"languages":[{"name":"English","url":"/datasets/language/english"}],"variants":["VQA-CE"],"data_loaders":[],"num_papers_in_archive":7,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28"},"benchmarks":[{"leaderboard":"/sota/visual-question-answering-on-vqa-ce","task":"Visual Question Answering (VQA)","dataset_variant":"VQA-CE","rows":9,"metrics":["Accuracy (Counterexamples)"],"first_row_in_archive_order":{"model":"RandImg","paper":"/paper/beyond-question-based-biases-assessing","metrics":{"Accuracy (Counterexamples)":"34.41"},"code_links":[{"title":"cdancette/detect-shortcuts","url":"https://github.com/cdancette/detect-shortcuts"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"}],"papers_with_a_benchmark_row":[{"paper":"/paper/beyond-question-based-biases-assessing","title":"Beyond Question-Based Biases: Assessing Multimodal Shortcut Learning in Visual Question Answering","date":"2021-04-07","rows_on_this_dataset":9,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":2,"samples_ran":1,"samples_unverified":1,"pointer_only_for_licence":2,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}}],"syntology_totals":{"read_at":"2026-09-24T18:15:14+00:00","papers_with_samples":1,"samples_harvested":2,"samples_ran":1,"samples_unverified":1,"pointer_only_for_licence":2,"papers_with_no_sample_that_ran":0,"note":"the per-paper counts above, summed; not a rate"},"papers_note":"The archive never published its papers-using-dataset list; these are papers with a leaderboard row on this dataset's benchmarks."}