{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/surgical-vqla-adversarial-contrastive","title":"Surgical-VQLA++: Adversarial Contrastive Learning for Calibrated Robust Visual Question-Localized Answering in Robotic Surgery","arxiv_id":"2408.04958","date":"2024-08-09","proceeding":null,"authors":["Long Bai","Guankun Wang","Mobarakol Islam","Lalithkumar Seenivasan","An Wang","Hongliang Ren"],"abstract":"Medical visual question answering (VQA) bridges the gap between visual information and clinical decision-making, enabling doctors to extract understanding from clinical images and videos. In particular, surgical VQA can enhance the interpretation of surgical data, aiding in accurate diagnoses, effective education, and clinical interventions. However, the inability of VQA models to visually indicate the regions of interest corresponding to the given questions results in incomplete comprehension of the surgical scene. To tackle this, we propose the surgical visual question localized-answering (VQLA) for precise and context-aware responses to specific queries regarding surgical images. Furthermore, to address the strong demand for safety in surgical scenarios and potential corruptions in image acquisition and transmission, we propose a novel approach called Calibrated Co-Attention Gated Vision-Language (C$^2$G-ViL) embedding to integrate and align multimodal information effectively. Additionally, we leverage the adversarial sample-based contrastive learning strategy to boost our performance and robustness. We also extend our EndoVis-18-VQLA and EndoVis-17-VQLA datasets to broaden the scope and application of our data. Extensive experiments on the aforementioned datasets demonstrate the remarkable performance and robustness of our solution. Our solution can effectively combat real-world image corruption. Thus, our proposed approach can serve as an effective tool for assisting surgical education, patient care, and enhancing surgical outcomes.","url_abs":"https://arxiv.org/abs/2408.04958v2","url_pdf":"https://arxiv.org/pdf/2408.04958v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"surgical-vqla-adversarial-contrastive","repo_url":"https://github.com/longbai1006/surgical-vqlaplus","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"contrastive-learning","task_name":"Contrastive Learning"},{"task_slug":"medical-visual-question-answering","task_name":"Medical Visual Question Answering"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"},{"task_slug":"visual-question-answering","task_name":"Visual Question Answering (VQA)"}],"methods":[{"method_slug":"align","method_name":"ALIGN"},{"method_slug":"contrastive-learning","method_name":"Contrastive Learning"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2408.04958","atlas_url":"https://app.syntology.ai/?focus=2408.04958","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2408.04958"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/longbai1006/surgical-vqlaplus","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":3,"ran_draft_wrong":1},"by_repo_kind":{"official":{"samples":4,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"53daad1d7b99fbdf","entry":"accuracy","repo":"longbai1006/surgical-vqlaplus","repo_kind":"official","path":"utils.py","file_url":"https://github.com/longbai1006/surgical-vqlaplus/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"53daad1d7b99fbdf"}},{"code_sha256_prefix":"19988d5d6c4a4d7c","entry":"calc_acc","repo":"longbai1006/surgical-vqlaplus","repo_kind":"official","path":"utils.py","file_url":"https://github.com/longbai1006/surgical-vqlaplus/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"19988d5d6c4a4d7c"}},{"code_sha256_prefix":"9fd07c1d7cd49c46","entry":"calc_classwise_acc","repo":"longbai1006/surgical-vqlaplus","repo_kind":"official","path":"utils.py","file_url":"https://github.com/longbai1006/surgical-vqlaplus/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9fd07c1d7cd49c46"}},{"code_sha256_prefix":"e766a0a3063c15d0","entry":"make_mask","repo":"longbai1006/surgical-vqlaplus","repo_kind":"official","path":"models/CATViLEmbedding.py","file_url":"https://github.com/longbai1006/surgical-vqlaplus/blob/HEAD/models/CATViLEmbedding.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e766a0a3063c15d0"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}