{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/seeing-is-believing-mitigating-hallucination","title":"Seeing is Believing: Mitigating Hallucination in Large Vision-Language Models via CLIP-Guided Decoding","arxiv_id":"2402.15300","date":"2024-02-23","proceeding":null,"authors":["Ailin Deng","Zhirui Chen","Bryan Hooi"],"abstract":"Large Vision-Language Models (LVLMs) are susceptible to object hallucinations, an issue in which their generated text contains non-existent objects, greatly limiting their reliability and practicality. Current approaches often rely on the model's token likelihoods or other internal information, instruction tuning on additional datasets, or incorporating complex external tools. We first perform empirical analysis on sentence-level LVLM hallucination, finding that CLIP similarity to the image acts as a stronger and more robust indicator of hallucination compared to token likelihoods. Motivated by this, we introduce our CLIP-Guided Decoding (CGD) approach, a straightforward but effective training-free approach to reduce object hallucination at decoding time. CGD uses CLIP to guide the model's decoding process by enhancing visual grounding of generated text with the image. Experiments demonstrate that CGD effectively mitigates object hallucination across multiple LVLM families while preserving the utility of text generation. Codes are available at https://github.com/d-ailin/CLIP-Guided-Decoding.","url_abs":"https://arxiv.org/abs/2402.15300v2","url_pdf":"https://arxiv.org/pdf/2402.15300v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"seeing-is-believing-mitigating-hallucination","repo_url":"https://github.com/d-ailin/clip-guided-decoding","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"jax","reach":{"status":"ok"}},{"paper_slug":"seeing-is-believing-mitigating-hallucination","repo_url":"https://github.com/d-ailin/blind-faith-in-text","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"hallucination","task_name":"Hallucination"},{"task_slug":"object","task_name":"Object"},{"task_slug":"object-hallucination","task_name":"Object Hallucination"},{"task_slug":"sentence","task_name":"Sentence"},{"task_slug":"text-generation","task_name":"Text Generation"},{"task_slug":"visual-grounding","task_name":"Visual Grounding"}],"methods":[{"method_slug":"clip","method_name":"CLIP"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2402.15300","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.15300"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/d-ailin/blind-faith-in-text","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/d-ailin/clip-guided-decoding","reach":{"status":"ok"}}],"summary":{"ran":11,"unverified":6},"by_repo_kind":{"official":{"samples":17,"ran":11,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":17,"samples":[{"code_sha256_prefix":"17fcb964a86c9b96","entry":"blip_generate","repo":"d-ailin/clip-guided-decoding","repo_kind":"official","path":"model_utils/blip2_vicuna_instruct.py","file_url":"https://github.com/d-ailin/clip-guided-decoding/blob/HEAD/model_utils/blip2_vicuna_instruct.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"17fcb964a86c9b96"}},{"code_sha256_prefix":"3f348347bb8aeb3d","entry":"check_if_terminal","repo":"d-ailin/clip-guided-decoding","repo_kind":"official","path":"gen/clip_guided.py","file_url":"https://github.com/d-ailin/clip-guided-decoding/blob/HEAD/gen/clip_guided.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3f348347bb8aeb3d"}},{"code_sha256_prefix":"d3ba5cc0b17533fb","entry":"check_if_termindal_w_tokens","repo":"d-ailin/clip-guided-decoding","repo_kind":"official","path":"gen/clip_guided.py","file_url":"https://github.com/d-ailin/clip-guided-decoding/blob/HEAD/gen/clip_guided.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d3ba5cc0b17533fb"}},{"code_sha256_prefix":"928cc991142557ee","entry":"create_question","repo":"d-ailin/clip-guided-decoding","repo_kind":"official","path":"lib/data_utils.py","file_url":"https://github.com/d-ailin/clip-guided-decoding/blob/HEAD/lib/data_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"928cc991142557ee"}},{"code_sha256_prefix":"26edd08622622687","entry":"get_all_sentence_indexs","repo":"d-ailin/clip-guided-decoding","repo_kind":"official","path":"model_utils/data.py","file_url":"https://github.com/d-ailin/clip-guided-decoding/blob/HEAD/model_utils/data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"26edd08622622687"}},{"code_sha256_prefix":"47ffe9a4a13a2371","entry":"get_last_sentence_indexs","repo":"d-ailin/clip-guided-decoding","repo_kind":"official","path":"model_utils/data.py","file_url":"https://github.com/d-ailin/clip-guided-decoding/blob/HEAD/model_utils/data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"47ffe9a4a13a2371"}},{"code_sha256_prefix":"92d3ced653e2c9d6","entry":"model_generate_anw","repo":"d-ailin/clip-guided-decoding","repo_kind":"official","path":"lib/utils.py","file_url":"https://github.com/d-ailin/clip-guided-decoding/blob/HEAD/lib/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"92d3ced653e2c9d6"}},{"code_sha256_prefix":"76b08f5dd16d6980","entry":"model_input_tpl_format","repo":"d-ailin/clip-guided-decoding","repo_kind":"official","path":"lib/utils.py","file_url":"https://github.com/d-ailin/clip-guided-decoding/blob/HEAD/lib/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"76b08f5dd16d6980"}},{"code_sha256_prefix":"1b25045b752f96ec","entry":"prepare_input","repo":"d-ailin/clip-guided-decoding","repo_kind":"official","path":"model_utils/blip2_vicuna_instruct.py","file_url":"https://github.com/d-ailin/clip-guided-decoding/blob/HEAD/model_utils/blip2_vicuna_instruct.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1b25045b752f96ec"}},{"code_sha256_prefix":"a321878fe42f6e2b","entry":"prepare_sampling_params","repo":"d-ailin/clip-guided-decoding","repo_kind":"official","path":"gen/rsp_sampling.py","file_url":"https://github.com/d-ailin/clip-guided-decoding/blob/HEAD/gen/rsp_sampling.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a321878fe42f6e2b"}},{"code_sha256_prefix":"7075ec400b8750ce","entry":"rsp_sampling","repo":"d-ailin/clip-guided-decoding","repo_kind":"official","path":"gen/rsp_sampling.py","file_url":"https://github.com/d-ailin/clip-guided-decoding/blob/HEAD/gen/rsp_sampling.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7075ec400b8750ce"}},{"code_sha256_prefix":"aa2ef0873fff54f0","entry":"generate_func","repo":"d-ailin/clip-guided-decoding","repo_kind":"official","path":"model_utils/blip2_vicuna_instruct.py","file_url":"https://github.com/d-ailin/clip-guided-decoding/blob/HEAD/model_utils/blip2_vicuna_instruct.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"aa2ef0873fff54f0"}},{"code_sha256_prefix":"d9bd8ba1bdcc4c5e","entry":"get_save_paths","repo":"d-ailin/clip-guided-decoding","repo_kind":"official","path":"lib/data_utils.py","file_url":"https://github.com/d-ailin/clip-guided-decoding/blob/HEAD/lib/data_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d9bd8ba1bdcc4c5e"}},{"code_sha256_prefix":"3831fed68df781a6","entry":"get_synonyms","repo":"d-ailin/clip-guided-decoding","repo_kind":"official","path":"eval_main.py","file_url":"https://github.com/d-ailin/clip-guided-decoding/blob/HEAD/eval_main.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3831fed68df781a6"}},{"code_sha256_prefix":"5d60a6d44891d2bb","entry":"is_end","repo":"d-ailin/clip-guided-decoding","repo_kind":"official","path":"gen/clip_guided.py","file_url":"https://github.com/d-ailin/clip-guided-decoding/blob/HEAD/gen/clip_guided.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5d60a6d44891d2bb"}},{"code_sha256_prefix":"ac4396da119cebfe","entry":"load_model","repo":"d-ailin/clip-guided-decoding","repo_kind":"official","path":"lib/utils.py","file_url":"https://github.com/d-ailin/clip-guided-decoding/blob/HEAD/lib/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ac4396da119cebfe"}},{"code_sha256_prefix":"d7192af5de6bc855","entry":"turn_token_score_in_word","repo":"d-ailin/clip-guided-decoding","repo_kind":"official","path":"lib/data_utils.py","file_url":"https://github.com/d-ailin/clip-guided-decoding/blob/HEAD/lib/data_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d7192af5de6bc855"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}