{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/mitigating-hallucinations-in-large-vision","title":"Mitigating Hallucinations in Large Vision-Language Models with Instruction Contrastive Decoding","arxiv_id":"2403.18715","date":"2024-03-27","proceeding":null,"authors":["Xintong Wang","Jingheng Pan","Liang Ding","Chris Biemann"],"abstract":"Large Vision-Language Models (LVLMs) are increasingly adept at generating contextually detailed and coherent responses from visual inputs. However, their application in multimodal decision-making and open-ended generation is hindered by a notable rate of hallucinations, where generated text inaccurately represents the visual contents. To address this issue, this paper introduces the Instruction Contrastive Decoding (ICD) method, a novel approach designed to reduce hallucinations during LVLM inference. Our method is inspired by our observation that what we call disturbance instructions significantly exacerbate hallucinations in multimodal fusion modules. ICD contrasts distributions from standard and instruction disturbance, thereby increasing alignment uncertainty and effectively subtracting hallucinated concepts from the original distribution. Through comprehensive experiments on discriminative benchmarks (POPE and MME) and a generative benchmark (LLaVa-Bench), we demonstrate that ICD significantly mitigates both object-level and attribute-level hallucinations. Moreover, our method not only addresses hallucinations but also significantly enhances the general perception and recognition capabilities of LVLMs.","url_abs":"https://arxiv.org/abs/2403.18715v2","url_pdf":"https://arxiv.org/pdf/2403.18715v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"mitigating-hallucinations-in-large-vision","repo_url":"https://github.com/p1k0pan/ICD","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"mitigating-hallucinations-in-large-vision","repo_url":"https://github.com/huofushuo/SID","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":null}],"tasks":[{"task_slug":"attribute","task_name":"Attribute"},{"task_slug":"decision-making","task_name":"Decision Making"},{"task_slug":"mme","task_name":"MME"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2403.18715","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.18715"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/p1k0pan/ICD","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/huofushuo/SID","reach":null}],"summary":{"ran_draft_wrong":1,"unverified":7},"by_repo_kind":{"official":{"samples":7,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":1,"samples":[{"code_sha256_prefix":"a06c0813244b6279","entry":"recorder","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"a06c0813244b6279"}},{"code_sha256_prefix":"fe66de066c32d0fc","entry":"add_diffusion_noise","repo":"p1k0pan/ICD","repo_kind":"official","path":"icd_utils/vcd_add_noise.py","file_url":"https://github.com/p1k0pan/ICD/blob/HEAD/icd_utils/vcd_add_noise.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fe66de066c32d0fc"}},{"code_sha256_prefix":"3bd270aa6cbe1ca4","entry":"icd","repo":"p1k0pan/ICD","repo_kind":"official","path":"experiments/gen_scripts/icd_coco_chair.py","file_url":"https://github.com/p1k0pan/ICD/blob/HEAD/experiments/gen_scripts/icd_coco_chair.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3bd270aa6cbe1ca4"}},{"code_sha256_prefix":"600d7db618c94fe9","entry":"icd","repo":"p1k0pan/ICD","repo_kind":"official","path":"experiments/gen_scripts/icd_ib_co.py","file_url":"https://github.com/p1k0pan/ICD/blob/HEAD/experiments/gen_scripts/icd_ib_co.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"600d7db618c94fe9"}},{"code_sha256_prefix":"523ca1ea0ef669eb","entry":"icd","repo":"p1k0pan/ICD","repo_kind":"official","path":"experiments/gen_scripts/icd_ib_ok_vqa.py","file_url":"https://github.com/p1k0pan/ICD/blob/HEAD/experiments/gen_scripts/icd_ib_ok_vqa.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"523ca1ea0ef669eb"}},{"code_sha256_prefix":"9b8248f96b730ec2","entry":"icd","repo":"p1k0pan/ICD","repo_kind":"official","path":"experiments/gen_scripts/icd_ib_pope.py","file_url":"https://github.com/p1k0pan/ICD/blob/HEAD/experiments/gen_scripts/icd_ib_pope.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9b8248f96b730ec2"}},{"code_sha256_prefix":"a557c858e7f610f3","entry":"sample","repo":"p1k0pan/ICD","repo_kind":"official","path":"icd_utils/icd_sample.py","file_url":"https://github.com/p1k0pan/ICD/blob/HEAD/icd_utils/icd_sample.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a557c858e7f610f3"}},{"code_sha256_prefix":"a975dbf49164bd1d","entry":"sample","repo":"p1k0pan/ICD","repo_kind":"official","path":"icd_utils/icd_sample_llava.py","file_url":"https://github.com/p1k0pan/ICD/blob/HEAD/icd_utils/icd_sample_llava.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a975dbf49164bd1d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}