{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/volcano-mitigating-multimodal-hallucination","title":"Volcano: Mitigating Multimodal Hallucination through Self-Feedback Guided Revision","arxiv_id":"2311.07362","date":"2023-11-13","proceeding":null,"authors":["Seongyun Lee","Sue Hyun Park","Yongrae Jo","Minjoon Seo"],"abstract":"Large multimodal models suffer from multimodal hallucination, where they provide incorrect responses misaligned with the given visual information. Recent works have conjectured that one of the reasons behind multimodal hallucination is due to the vision encoder failing to ground on the image properly. To mitigate this issue, we propose a novel approach that leverages self-feedback as visual cues. Building on this approach, we introduce Volcano, a multimodal self-feedback guided revision model. Volcano generates natural language feedback to its initial response based on the provided visual information and utilizes this feedback to self-revise its initial response. Volcano effectively reduces multimodal hallucination and achieves state-of-the-art on MMHal-Bench, POPE, and GAVIE. It also improves on general multimodal abilities and outperforms previous models on MM-Vet and MMBench. Through qualitative analysis, we show that Volcano's feedback is properly grounded on the image than the initial response. This indicates that Volcano can provide itself with richer visual information through feedback generation, leading to self-correct hallucinations. We publicly release our model, data, and code at https://github.com/kaistAI/Volcano}{github.com/kaistAI/Volcano","url_abs":"https://arxiv.org/abs/2311.07362v4","url_pdf":"https://arxiv.org/pdf/2311.07362v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"volcano-mitigating-multimodal-hallucination","repo_url":"https://github.com/kaistai/volcano","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"hallucination","task_name":"Hallucination"},{"task_slug":null,"task_name":"MM-Vet"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/visual-question-answering-on-mm-vet","task":"Visual Question Answering","dataset":"MM-Vet","model":"VOLCANO 13B","rank_in_archive_order":125,"of":231,"metrics":{"GPT-4 score":"38.0","Params":"13B"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-mm-vet","task":"Visual Question Answering","dataset":"MM-Vet","model":"VOLCANO 7B","rank_in_archive_order":186,"of":231,"metrics":{"GPT-4 score":"32.0","Params":"7B"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2311.07362","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.07362"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kaistai/volcano","reach":{"status":"ok"}}],"summary":{"ran_draft_wrong":3,"ran":3,"ran_violates":1,"ran_honours":1,"ran_fixture":1,"unverified":1},"by_repo_kind":{"official":{"samples":10,"ran":9,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":10,"samples":[{"code_sha256_prefix":"20e4f665698a3d18","entry":"collate_fn","repo":"kaistai/volcano","repo_kind":"official","path":"llava/eval/model_vqa_loader.py","file_url":"https://github.com/kaistai/volcano/blob/HEAD/llava/eval/model_vqa_loader.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"20e4f665698a3d18"}},{"code_sha256_prefix":"7e03b180fa317c9a","entry":"divide_to_patches","repo":"kaistai/volcano","repo_kind":"official","path":"llava/mm_utils.py","file_url":"https://github.com/kaistai/volcano/blob/HEAD/llava/mm_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7e03b180fa317c9a"}},{"code_sha256_prefix":"03d4ba90a6885914","entry":"fuse_image_attentions_for_image_features","repo":"kaistai/volcano","repo_kind":"official","path":"llava/visualize/run_volcano_with_image_attention.py","file_url":"https://github.com/kaistai/volcano/blob/HEAD/llava/visualize/run_volcano_with_image_attention.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"03d4ba90a6885914"}},{"code_sha256_prefix":"42a46570620cd9fa","entry":"get_chunk","repo":"kaistai/volcano","repo_kind":"official","path":"llava/eval/volcano_pope.py","file_url":"https://github.com/kaistai/volcano/blob/HEAD/llava/eval/volcano_pope.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"42a46570620cd9fa"}},{"code_sha256_prefix":"bae18947b56f2be1","entry":"is_none","repo":"kaistai/volcano","repo_kind":"official","path":"llava/eval/model_vqa_mmbench.py","file_url":"https://github.com/kaistai/volcano/blob/HEAD/llava/eval/model_vqa_mmbench.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bae18947b56f2be1"}},{"code_sha256_prefix":"9b3c1cb391672ccb","entry":"load_image","repo":"kaistai/volcano","repo_kind":"official","path":"llava/eval/volcano_gavie.py","file_url":"https://github.com/kaistai/volcano/blob/HEAD/llava/eval/volcano_gavie.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9b3c1cb391672ccb"}},{"code_sha256_prefix":"468eedeba67f1b00","entry":"resize_and_pad_image","repo":"kaistai/volcano","repo_kind":"official","path":"llava/mm_utils.py","file_url":"https://github.com/kaistai/volcano/blob/HEAD/llava/mm_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"468eedeba67f1b00"}},{"code_sha256_prefix":"3999ff487573f32c","entry":"select_best_resolution","repo":"kaistai/volcano","repo_kind":"official","path":"llava/mm_utils.py","file_url":"https://github.com/kaistai/volcano/blob/HEAD/llava/mm_utils.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3999ff487573f32c"}},{"code_sha256_prefix":"076c252c52cbb161","entry":"split_list","repo":"kaistai/volcano","repo_kind":"official","path":"llava/eval/volcano_pope.py","file_url":"https://github.com/kaistai/volcano/blob/HEAD/llava/eval/volcano_pope.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"076c252c52cbb161"}},{"code_sha256_prefix":"18e694aa6960fd03","entry":"extract_image_features","repo":"kaistai/volcano","repo_kind":"official","path":"llava/eval/volcano_gavie.py","file_url":"https://github.com/kaistai/volcano/blob/HEAD/llava/eval/volcano_gavie.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"18e694aa6960fd03"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}