{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/mitigating-object-hallucinations-in-large","title":"Mitigating Object Hallucinations in Large Vision-Language Models through Visual Contrastive Decoding","arxiv_id":"2311.16922","date":"2023-11-28","proceeding":"CVPR 2024 1","authors":["Sicong Leng","Hang Zhang","Guanzheng Chen","Xin Li","Shijian Lu","Chunyan Miao","Lidong Bing"],"abstract":"Large Vision-Language Models (LVLMs) have advanced considerably, intertwining visual recognition and language understanding to generate content that is not only coherent but also contextually attuned. Despite their success, LVLMs still suffer from the issue of object hallucinations, where models generate plausible yet incorrect outputs that include objects that do not exist in the images. To mitigate this issue, we introduce Visual Contrastive Decoding (VCD), a simple and training-free method that contrasts output distributions derived from original and distorted visual inputs. The proposed VCD effectively reduces the over-reliance on statistical bias and unimodal priors, two essential causes of object hallucinations. This adjustment ensures the generated content is closely grounded to visual inputs, resulting in contextually accurate outputs. Our experiments show that VCD, without either additional training or the usage of external tools, significantly mitigates the object hallucination issue across different LVLM families. Beyond mitigating object hallucinations, VCD also excels in general LVLM benchmarks, highlighting its wide-ranging applicability.","url_abs":"https://arxiv.org/abs/2311.16922v1","url_pdf":"https://arxiv.org/pdf/2311.16922v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"mitigating-object-hallucinations-in-large","repo_url":"https://github.com/bradyfu/awesome-multimodal-large-language-models","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"mitigating-object-hallucinations-in-large","repo_url":"https://github.com/damo-nlp-sg/vcd","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"mitigating-object-hallucinations-in-large","repo_url":"https://github.com/damo-nlp-sg/inf-clip","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"mitigating-object-hallucinations-in-large","repo_url":"https://github.com/huofushuo/SID","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":null},{"paper_slug":"mitigating-object-hallucinations-in-large","repo_url":"https://github.com/p1k0pan/ICD","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"mitigating-object-hallucinations-in-large","repo_url":"https://github.com/xinding-sys/StreamMind","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"mitigating-object-hallucinations-in-large","repo_url":"https://github.com/ziwei-zheng/nullu","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"hallucination","task_name":"Hallucination"},{"task_slug":"object","task_name":"Object"},{"task_slug":"object-hallucination","task_name":"Object Hallucination"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2311.16922","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.16922"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/p1k0pan/ICD","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/damo-nlp-sg/vcd","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ziwei-zheng/nullu","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/huofushuo/SID","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/damo-nlp-sg/inf-clip","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/xinding-sys/StreamMind","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/bradyfu/awesome-multimodal-large-language-models","reach":{"status":"ok"}}],"summary":{"ran":5,"ran_draft_wrong":4,"unverified":5},"by_repo_kind":{"official":{"samples":13,"ran":8,"repositories":1},"listed":{"samples":1,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":1,"samples":[{"code_sha256_prefix":"77e8a3ac46f3afec","entry":"get_2d_sincos_pos_embed","repo":"damo-nlp-sg/vcd","repo_kind":"official","path":"experiments/Qwen_VL/visual.py","file_url":"https://github.com/damo-nlp-sg/vcd/blob/HEAD/experiments/Qwen_VL/visual.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"77e8a3ac46f3afec"}},{"code_sha256_prefix":"50c4398a8645438a","entry":"get_abs_pos","repo":"damo-nlp-sg/vcd","repo_kind":"official","path":"experiments/Qwen_VL/visual.py","file_url":"https://github.com/damo-nlp-sg/vcd/blob/HEAD/experiments/Qwen_VL/visual.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"50c4398a8645438a"}},{"code_sha256_prefix":"d2c4545e71293dec","entry":"get_batch","repo":"damo-nlp-sg/vcd","repo_kind":"official","path":"experiments/Qwen_VL/qwen_generation_utils.py","file_url":"https://github.com/damo-nlp-sg/vcd/blob/HEAD/experiments/Qwen_VL/qwen_generation_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d2c4545e71293dec"}},{"code_sha256_prefix":"42a46570620cd9fa","entry":"get_chunk","repo":"damo-nlp-sg/vcd","repo_kind":"official","path":"experiments/eval/object_hallucination_vqa_instructblip.py","file_url":"https://github.com/damo-nlp-sg/vcd/blob/HEAD/experiments/eval/object_hallucination_vqa_instructblip.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"42a46570620cd9fa"}},{"code_sha256_prefix":"2d28c844d3557fe6","entry":"get_ltor_masks_and_position_ids","repo":"damo-nlp-sg/vcd","repo_kind":"official","path":"experiments/Qwen_VL/qwen_generation_utils.py","file_url":"https://github.com/damo-nlp-sg/vcd/blob/HEAD/experiments/Qwen_VL/qwen_generation_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2d28c844d3557fe6"}},{"code_sha256_prefix":"a83d13d0950d1a06","entry":"pad_batch","repo":"damo-nlp-sg/vcd","repo_kind":"official","path":"experiments/Qwen_VL/qwen_generation_utils.py","file_url":"https://github.com/damo-nlp-sg/vcd/blob/HEAD/experiments/Qwen_VL/qwen_generation_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a83d13d0950d1a06"}},{"code_sha256_prefix":"a06c0813244b6279","entry":"recorder","repo":"huofushuo/SID","repo_kind":"listed","path":"pope_eval.py","file_url":"https://github.com/huofushuo/SID/blob/HEAD/pope_eval.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a06c0813244b6279"}},{"code_sha256_prefix":"076c252c52cbb161","entry":"split_list","repo":"damo-nlp-sg/vcd","repo_kind":"official","path":"experiments/eval/object_hallucination_vqa_instructblip.py","file_url":"https://github.com/damo-nlp-sg/vcd/blob/HEAD/experiments/eval/object_hallucination_vqa_instructblip.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"076c252c52cbb161"}},{"code_sha256_prefix":"cb33571427334815","entry":"tile","repo":"damo-nlp-sg/vcd","repo_kind":"official","path":"experiments/lavis/models/base_model.py","file_url":"https://github.com/damo-nlp-sg/vcd/blob/HEAD/experiments/lavis/models/base_model.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cb33571427334815"}},{"code_sha256_prefix":"fe66de066c32d0fc","entry":"add_diffusion_noise","repo":"damo-nlp-sg/vcd","repo_kind":"official","path":"vcd_utils/vcd_add_noise.py","file_url":"https://github.com/damo-nlp-sg/vcd/blob/HEAD/vcd_utils/vcd_add_noise.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fe66de066c32d0fc"}},{"code_sha256_prefix":"0ec9fc2025c16f65","entry":"all_gather_with_grad","repo":"damo-nlp-sg/vcd","repo_kind":"official","path":"experiments/lavis/models/base_model.py","file_url":"https://github.com/damo-nlp-sg/vcd/blob/HEAD/experiments/lavis/models/base_model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0ec9fc2025c16f65"}},{"code_sha256_prefix":"fbda921818ec02df","entry":"apply_rotary_pos_emb","repo":"damo-nlp-sg/vcd","repo_kind":"official","path":"experiments/Qwen_VL/modeling_qwen.py","file_url":"https://github.com/damo-nlp-sg/vcd/blob/HEAD/experiments/Qwen_VL/modeling_qwen.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fbda921818ec02df"}},{"code_sha256_prefix":"2884deadbb637c3e","entry":"get_2d_sincos_pos_embed_from_grid","repo":"damo-nlp-sg/vcd","repo_kind":"official","path":"experiments/Qwen_VL/visual.py","file_url":"https://github.com/damo-nlp-sg/vcd/blob/HEAD/experiments/Qwen_VL/visual.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2884deadbb637c3e"}},{"code_sha256_prefix":"265b8739e27fe5c8","entry":"sample","repo":"damo-nlp-sg/vcd","repo_kind":"official","path":"vcd_utils/vcd_sample.py","file_url":"https://github.com/damo-nlp-sg/vcd/blob/HEAD/vcd_utils/vcd_sample.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"265b8739e27fe5c8"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}