{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/omnimedvqa-a-new-large-scale-comprehensive","title":"OmniMedVQA: A New Large-Scale Comprehensive Evaluation Benchmark for Medical LVLM","arxiv_id":"2402.09181","date":"2024-02-14","proceeding":"CVPR 2024 1","authors":["Yutao Hu","Tianbin Li","Quanfeng Lu","Wenqi Shao","Junjun He","Yu Qiao","Ping Luo"],"abstract":"Large Vision-Language Models (LVLMs) have demonstrated remarkable capabilities in various multimodal tasks. However, their potential in the medical domain remains largely unexplored. A significant challenge arises from the scarcity of diverse medical images spanning various modalities and anatomical regions, which is essential in real-world medical applications. To solve this problem, in this paper, we introduce OmniMedVQA, a novel comprehensive medical Visual Question Answering (VQA) benchmark. This benchmark is collected from 73 different medical datasets, including 12 different modalities and covering more than 20 distinct anatomical regions. Importantly, all images in this benchmark are sourced from authentic medical scenarios, ensuring alignment with the requirements of the medical field and suitability for evaluating LVLMs. Through our extensive experiments, we have found that existing LVLMs struggle to address these medical VQA problems effectively. Moreover, what surprises us is that medical-specialized LVLMs even exhibit inferior performance to those general-domain models, calling for a more versatile and robust LVLM in the biomedical field. The evaluation results not only reveal the current limitations of LVLM in understanding real medical images but also highlight our dataset's significance. Our code with dataset are available at https://github.com/OpenGVLab/Multi-Modality-Arena.","url_abs":"https://arxiv.org/abs/2402.09181v2","url_pdf":"https://arxiv.org/pdf/2402.09181v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"omnimedvqa-a-new-large-scale-comprehensive","repo_url":"https://github.com/opengvlab/multi-modality-arena","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"medical-visual-question-answering","task_name":"Medical Visual Question Answering"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"},{"task_slug":"visual-question-answering","task_name":"Visual Question Answering (VQA)"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2402.09181","atlas_url":"https://app.syntology.ai/?focus=2402.09181","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.09181"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/opengvlab/multi-modality-arena","reach":{"status":"ok"}}],"summary":{"ran_fixture":2,"ran":3,"ran_draft_wrong":1,"ran_violates":1,"unverified":1},"by_repo_kind":{"official":{"samples":8,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":8,"samples":[{"code_sha256_prefix":"b47d48e431b34acd","entry":"apply_rotary_emb","repo":"opengvlab/multi-modality-arena","repo_kind":"official","path":"LVLM_evaluation/Multi_turn_Reasoning/lib/llama.py","file_url":"https://github.com/opengvlab/multi-modality-arena/blob/HEAD/LVLM_evaluation/Multi_turn_Reasoning/lib/llama.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b47d48e431b34acd"}},{"code_sha256_prefix":"9736f07175994fde","entry":"apply_rotary_emb","repo":"opengvlab/multi-modality-arena","repo_kind":"official","path":"LVLM_evaluation/Multi_turn_Reasoning/LLaMA-Adapter-v2/models_mae.py","file_url":"https://github.com/opengvlab/multi-modality-arena/blob/HEAD/LVLM_evaluation/Multi_turn_Reasoning/LLaMA-Adapter-v2/models_mae.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9736f07175994fde"}},{"code_sha256_prefix":"60341abc605d8898","entry":"get_chat","repo":"opengvlab/multi-modality-arena","repo_kind":"official","path":"tiny_lvlm_evaluation/multi_turn.py","file_url":"https://github.com/opengvlab/multi-modality-arena/blob/HEAD/tiny_lvlm_evaluation/multi_turn.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"60341abc605d8898"}},{"code_sha256_prefix":"ae0020125496f261","entry":"get_model_name","repo":"opengvlab/multi-modality-arena","repo_kind":"official","path":"LVLM_evaluation/Multi_turn_Reasoning/lib/llava_lib.py","file_url":"https://github.com/opengvlab/multi-modality-arena/blob/HEAD/LVLM_evaluation/Multi_turn_Reasoning/lib/llava_lib.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ae0020125496f261"}},{"code_sha256_prefix":"791c72070c0b1cfb","entry":"node_to_dict","repo":"opengvlab/multi-modality-arena","repo_kind":"official","path":"LVLM_evaluation/Multi_turn_Reasoning/lib/minigpt4_config.py","file_url":"https://github.com/opengvlab/multi-modality-arena/blob/HEAD/LVLM_evaluation/Multi_turn_Reasoning/lib/minigpt4_config.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"791c72070c0b1cfb"}},{"code_sha256_prefix":"14a84c2cbfebc413","entry":"precompute_freqs_cis","repo":"opengvlab/multi-modality-arena","repo_kind":"official","path":"LVLM_evaluation/Multi_turn_Reasoning/LLaMA-Adapter-v2/models_mae.py","file_url":"https://github.com/opengvlab/multi-modality-arena/blob/HEAD/LVLM_evaluation/Multi_turn_Reasoning/LLaMA-Adapter-v2/models_mae.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"14a84c2cbfebc413"}},{"code_sha256_prefix":"70bf6ebaafd266c4","entry":"reshape_for_broadcast","repo":"opengvlab/multi-modality-arena","repo_kind":"official","path":"LVLM_evaluation/Multi_turn_Reasoning/LLaMA-Adapter-v2/models_mae.py","file_url":"https://github.com/opengvlab/multi-modality-arena/blob/HEAD/LVLM_evaluation/Multi_turn_Reasoning/LLaMA-Adapter-v2/models_mae.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"70bf6ebaafd266c4"}},{"code_sha256_prefix":"fb999e47877e08ff","entry":"load","repo":"opengvlab/multi-modality-arena","repo_kind":"official","path":"LVLM_evaluation/Multi_turn_Reasoning/lib/llama_adapter.py","file_url":"https://github.com/opengvlab/multi-modality-arena/blob/HEAD/LVLM_evaluation/Multi_turn_Reasoning/lib/llama_adapter.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fb999e47877e08ff"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}