{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/2408-02865","title":"VisionUnite: A Vision-Language Foundation Model for Ophthalmology Enhanced with Clinical Knowledge","arxiv_id":"2408.02865","date":"2024-08-05","proceeding":null,"authors":["Zihan Li","Diping Song","Zefeng Yang","Deming Wang","Fei Li","Xiulan Zhang","Paul E. Kinahan","Yu Qiao"],"abstract":"The need for improved diagnostic methods in ophthalmology is acute, especially in the less developed regions with limited access to specialists and advanced equipment. Therefore, we introduce VisionUnite, a novel vision-language foundation model for ophthalmology enhanced with clinical knowledge. VisionUnite has been pretrained on an extensive dataset comprising 1.24 million image-text pairs, and further refined using our proposed MMFundus dataset, which includes 296,379 high-quality fundus image-text pairs and 889,137 simulated doctor-patient dialogue instances. Our experiments indicate that VisionUnite outperforms existing generative foundation models such as GPT-4V and Gemini Pro. It also demonstrates diagnostic capabilities comparable to junior ophthalmologists. VisionUnite performs well in various clinical scenarios including open-ended multi-disease diagnosis, clinical explanation, and patient interaction, making it a highly versatile tool for initial ophthalmic disease screening. VisionUnite can also serve as an educational aid for junior ophthalmologists, accelerating their acquisition of knowledge regarding both common and rare ophthalmic conditions. VisionUnite represents a significant advancement in ophthalmology, with broad implications for diagnostics, medical education, and understanding of disease mechanisms.","url_abs":"https://arxiv.org/abs/2408.02865v1","url_pdf":"https://arxiv.org/pdf/2408.02865v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"2408-02865","repo_url":"https://github.com/HUANGLIZI/VisionUnite","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"clinical-knowledge","task_name":"Clinical Knowledge"},{"task_slug":"diagnostic","task_name":"Diagnostic"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2408.02865","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2408.02865"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/HUANGLIZI/VisionUnite","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_fixture":1,"ran":6,"ran_honours":1,"unverified":3},"by_repo_kind":{"official":{"samples":11,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"751b1c8ccbe160c4","entry":"cast_if_src_dtype","repo":"HUANGLIZI/VisionUnite","repo_kind":"official","path":"ImageBind/models/helpers.py","file_url":"https://github.com/HUANGLIZI/VisionUnite/blob/HEAD/ImageBind/models/helpers.py","link_basis":"plan_row","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"751b1c8ccbe160c4"}},{"code_sha256_prefix":"40c4752510219058","entry":"fold_dir","repo":"HUANGLIZI/VisionUnite","repo_kind":"official","path":"fundus_prep.py","file_url":"https://github.com/HUANGLIZI/VisionUnite/blob/HEAD/fundus_prep.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"40c4752510219058"}},{"code_sha256_prefix":"432c312da80537d6","entry":"get_clip_timepoints","repo":"HUANGLIZI/VisionUnite","repo_kind":"official","path":"ImageBind/data.py","file_url":"https://github.com/HUANGLIZI/VisionUnite/blob/HEAD/ImageBind/data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"432c312da80537d6"}},{"code_sha256_prefix":"4005aa64600585f3","entry":"get_mask_BZ","repo":"HUANGLIZI/VisionUnite","repo_kind":"official","path":"fundus_prep.py","file_url":"https://github.com/HUANGLIZI/VisionUnite/blob/HEAD/fundus_prep.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4005aa64600585f3"}},{"code_sha256_prefix":"35a743e50e17e66d","entry":"get_sinusoid_encoding_table","repo":"HUANGLIZI/VisionUnite","repo_kind":"official","path":"ImageBind/models/multimodal_preprocessors.py","file_url":"https://github.com/HUANGLIZI/VisionUnite/blob/HEAD/ImageBind/models/multimodal_preprocessors.py","link_basis":"plan_row","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"35a743e50e17e66d"}},{"code_sha256_prefix":"e76cb068e766a674","entry":"imread","repo":"HUANGLIZI/VisionUnite","repo_kind":"official","path":"fundus_prep.py","file_url":"https://github.com/HUANGLIZI/VisionUnite/blob/HEAD/fundus_prep.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e76cb068e766a674"}},{"code_sha256_prefix":"a703fea84ef259af","entry":"load_and_transform_vision_data","repo":"HUANGLIZI/VisionUnite","repo_kind":"official","path":"gradio_app.py","file_url":"https://github.com/HUANGLIZI/VisionUnite/blob/HEAD/gradio_app.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a703fea84ef259af"}},{"code_sha256_prefix":"e1e69d9ba9ffcb24","entry":"load_and_transform_vision_data","repo":"HUANGLIZI/VisionUnite","repo_kind":"official","path":"ImageBind/data.py","file_url":"https://github.com/HUANGLIZI/VisionUnite/blob/HEAD/ImageBind/data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e1e69d9ba9ffcb24"}},{"code_sha256_prefix":"308754348a33628a","entry":"imagebind_huge","repo":"HUANGLIZI/VisionUnite","repo_kind":"official","path":"ImageBind/models/imagebind_model.py","file_url":"https://github.com/HUANGLIZI/VisionUnite/blob/HEAD/ImageBind/models/imagebind_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"308754348a33628a"}},{"code_sha256_prefix":"71227efe6a7df1d2","entry":"interpolate_pos_encoding","repo":"HUANGLIZI/VisionUnite","repo_kind":"official","path":"ImageBind/models/multimodal_preprocessors.py","file_url":"https://github.com/HUANGLIZI/VisionUnite/blob/HEAD/ImageBind/models/multimodal_preprocessors.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"71227efe6a7df1d2"}},{"code_sha256_prefix":"ddd2e52e6b0956b5","entry":"interpolate_pos_encoding_2d","repo":"HUANGLIZI/VisionUnite","repo_kind":"official","path":"ImageBind/models/multimodal_preprocessors.py","file_url":"https://github.com/HUANGLIZI/VisionUnite/blob/HEAD/ImageBind/models/multimodal_preprocessors.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ddd2e52e6b0956b5"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}