{"url":"/dataset/vip-bench","name":"ViP-Bench","full_name":"Making Large Multimodal Models Understand Arbitrary Visual Prompts","description_markdown":"ViP-Bench is a comprehensive benchmark designed to assess the capability of multimodal models in understanding visual prompts across multiple dimensions. It aims to evaluate how well these models interpret various visual prompts, including recognition, OCR, knowledge, math, relationship reasoning, and language generation. ViP-Bench includes a diverse set of 303 images and questions, providing a thorough assessment of visual understanding capabilities at the region level. This benchmark sets a foundation for future research into multimodal models with arbitrary visual prompts.","description_withheld":null,"homepage":"https://vip-llava.github.io/","introduced_date":"2023-12-01","introduced_date_note":null,"introduced_by":{"paper":"/paper/making-large-multimodal-models-understand","title":"ViP-LLaVA: Making Large Multimodal Models Understand Arbitrary Visual Prompts","first_author":"Mu Cai","url":null},"license":{"name":"apache-2.0","url":"https://choosealicense.com/licenses/apache-2.0/"},"modalities":[{"name":"Images","url":"/datasets/modality/images"},{"name":"Texts","url":"/datasets/modality/texts"},{"name":"Interactive","url":"/datasets/modality/interactive"}],"tasks":[{"name":"Visual Question Answering","url":"/task/visual-question-answering-1","datasets_with_task":"/datasets/task/visual-question-answering-1"}],"languages":[{"name":"English","url":"/datasets/language/english"}],"variants":["ViP-Bench"],"data_loaders":[],"num_papers_in_archive":10,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28"},"benchmarks":[{"leaderboard":"/sota/visual-question-answering-on-vip-bench","task":"Visual Question Answering","dataset_variant":"ViP-Bench","rows":13,"metrics":["GPT-4 score (bbox)","GPT-4 score (human)"],"first_row_in_archive_order":{"model":"GPT-4V-turbo-detail:high (Visual Prompt)","paper":"/paper/gpt-4-technical-report-1","metrics":{"GPT-4 score (bbox)":"60.7","GPT-4 score (human)":"59.9"},"code_links":[{"title":"openai/evals","url":"https://github.com/openai/evals"},{"title":"shmsw25/factscore","url":"https://github.com/shmsw25/factscore"},{"title":"unispac/visual-adversarial-examples-jailbreak-large-language-models","url":"https://github.com/unispac/visual-adversarial-examples-jailbreak-large-language-models"},{"title":"gpt4life/alpagasus","url":"https://github.com/gpt4life/alpagasus"},{"title":"emrgnt-cmplxty/zero-shot-replication","url":"https://github.com/emrgnt-cmplxty/zero-shot-replication"},{"title":"ethz-privsec/superhuman-ai-consistency","url":"https://github.com/ethz-privsec/superhuman-ai-consistency"},{"title":"ethz-spylab/superhuman-ai-consistency","url":"https://github.com/ethz-spylab/superhuman-ai-consistency"},{"title":"eternityyw/tram-benchmark","url":"https://github.com/eternityyw/tram-benchmark"},{"title":"AUCOHL/RTL-Repo","url":"https://github.com/AUCOHL/RTL-Repo"},{"title":"zach-zhiling-zheng/reticular_chemist","url":"https://github.com/zach-zhiling-zheng/reticular_chemist"},{"title":"lflage/openfactscore","url":"https://github.com/lflage/openfactscore"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"}],"papers_with_a_benchmark_row":[{"paper":"/paper/inst-it-boosting-multimodal-instance","title":"Inst-IT: Boosting Multimodal Instance Understanding via Explicit Visual Prompt Instruction Tuning","date":"2024-12-04","rows_on_this_dataset":2,"code_links":1,"syntology":{"read_at":"2026-09-25T09:33:49+00:00","samples_harvested":1,"samples_ran":1,"samples_unverified":0,"pointer_only_for_licence":1,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/making-large-language-models-better-data","title":"Making Large Language Models Better Data Creators","date":"2023-10-31","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/improved-baselines-with-visual-instruction","title":"Improved Baselines with Visual Instruction Tuning","date":"2023-10-05","rows_on_this_dataset":2,"code_links":9,"syntology":{"read_at":"2026-09-25T09:33:49+00:00","samples_harvested":9,"samples_ran":6,"samples_unverified":3,"pointer_only_for_licence":8,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/qwen-vl-a-frontier-large-vision-language","title":"Qwen-VL: A Versatile Vision-Language Model for Understanding, Localization, Text Reading, and Beyond","date":"2023-08-24","rows_on_this_dataset":2,"code_links":2,"syntology":{"read_at":"2026-09-25T09:33:49+00:00","samples_harvested":2,"samples_ran":0,"samples_unverified":2,"pointer_only_for_licence":2,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/gpt4roi-instruction-tuning-large-language","title":"GPT4RoI: Instruction Tuning Large Language Model on Region-of-Interest","date":"2023-07-07","rows_on_this_dataset":1,"code_links":3,"syntology":{"read_at":"2026-09-25T09:33:49+00:00","samples_harvested":5,"samples_ran":3,"samples_unverified":2,"pointer_only_for_licence":5,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/shikra-unleashing-multimodal-llm-s","title":"Shikra: Unleashing Multimodal LLM's Referential Dialogue Magic","date":"2023-06-27","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-25T09:33:49+00:00","samples_harvested":2,"samples_ran":2,"samples_unverified":0,"pointer_only_for_licence":2,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/kosmos-2-grounding-multimodal-large-language","title":"Kosmos-2: Grounding Multimodal Large Language Models to the World","date":"2023-06-26","rows_on_this_dataset":1,"code_links":2,"syntology":null},{"paper":"/paper/instructblip-towards-general-purpose-vision","title":"InstructBLIP: Towards General-purpose Vision-Language Models with Instruction Tuning","date":"2023-05-11","rows_on_this_dataset":1,"code_links":4,"syntology":null},{"paper":"/paper/gpt-4-technical-report-1","title":"GPT-4 Technical Report","date":"2023-03-15","rows_on_this_dataset":2,"code_links":11,"syntology":{"read_at":"2026-09-25T09:33:49+00:00","samples_harvested":5,"samples_ran":5,"samples_unverified":0,"pointer_only_for_licence":1,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}}],"syntology_totals":{"read_at":"2026-09-25T09:33:49+00:00","papers_with_samples":6,"samples_harvested":24,"samples_ran":17,"samples_unverified":7,"pointer_only_for_licence":19,"papers_with_no_sample_that_ran":1,"note":"the per-paper counts above, summed; not a rate"},"papers_note":"The archive never published its papers-using-dataset list; these are papers with a leaderboard row on this dataset's benchmarks."}