{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/vl-uncertainty-detecting-hallucination-in","title":"VL-Uncertainty: Detecting Hallucination in Large Vision-Language Model via Uncertainty Estimation","arxiv_id":"2411.11919","date":"2024-11-18","proceeding":null,"authors":["Ruiyang Zhang","Hu Zhang","Zhedong Zheng"],"abstract":"Given the higher information load processed by large vision-language models (LVLMs) compared to single-modal LLMs, detecting LVLM hallucinations requires more human and time expense, and thus rise a wider safety concerns. In this paper, we introduce VL-Uncertainty, the first uncertainty-based framework for detecting hallucinations in LVLMs. Different from most existing methods that require ground-truth or pseudo annotations, VL-Uncertainty utilizes uncertainty as an intrinsic metric. We measure uncertainty by analyzing the prediction variance across semantically equivalent but perturbed prompts, including visual and textual data. When LVLMs are highly confident, they provide consistent responses to semantically equivalent queries. However, when uncertain, the responses of the target LVLM become more random. Considering semantically similar answers with different wordings, we cluster LVLM responses based on their semantic content and then calculate the cluster distribution entropy as the uncertainty measure to detect hallucination. Our extensive experiments on 10 LVLMs across four benchmarks, covering both free-form and multi-choice tasks, show that VL-Uncertainty significantly outperforms strong baseline methods in hallucination detection.","url_abs":"https://arxiv.org/abs/2411.11919v2","url_pdf":"https://arxiv.org/pdf/2411.11919v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"vl-uncertainty-detecting-hallucination-in","repo_url":"https://github.com/JT-Sun/Filtering-WoRA","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"hallucination","task_name":"Hallucination"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2411.11919","atlas_url":"https://app.syntology.ai/?focus=2411.11919","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2411.11919"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/JT-Sun/Filtering-WoRA","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_draft_wrong":4,"ran_fixture":1,"ran_honours":3,"ran":3,"unverified":3},"by_repo_kind":{"listed":{"samples":14,"ran":11,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"cf9ffa02a42184af","entry":"whitespace_tokenize","repo":"JT-Sun/Filtering-WoRA","repo_kind":"listed","path":"models/tokenization_bert.py","file_url":"https://github.com/JT-Sun/Filtering-WoRA/blob/HEAD/models/tokenization_bert.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":2,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cf9ffa02a42184af"}},{"code_sha256_prefix":"144d10b49baeb8a6","entry":"window_partition","repo":"JT-Sun/Filtering-WoRA","repo_kind":"listed","path":"models/swin_transformer.py","file_url":"https://github.com/JT-Sun/Filtering-WoRA/blob/HEAD/models/swin_transformer.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":2,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"144d10b49baeb8a6"}},{"code_sha256_prefix":"e0a06ded5d4f6c3c","entry":"box_cxcywh_to_xyxy","repo":"JT-Sun/Filtering-WoRA","repo_kind":"listed","path":"models/box_ops.py","file_url":"https://github.com/JT-Sun/Filtering-WoRA/blob/HEAD/models/box_ops.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e0a06ded5d4f6c3c"}},{"code_sha256_prefix":"ce749f837424dd8c","entry":"box_iou","repo":"JT-Sun/Filtering-WoRA","repo_kind":"listed","path":"models/box_ops.py","file_url":"https://github.com/JT-Sun/Filtering-WoRA/blob/HEAD/models/box_ops.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ce749f837424dd8c"}},{"code_sha256_prefix":"031b02fb990284ba","entry":"box_xyxy_to_cxcywh","repo":"JT-Sun/Filtering-WoRA","repo_kind":"listed","path":"models/box_ops.py","file_url":"https://github.com/JT-Sun/Filtering-WoRA/blob/HEAD/models/box_ops.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"031b02fb990284ba"}},{"code_sha256_prefix":"51e7514fd736cca3","entry":"build_mlp","repo":"JT-Sun/Filtering-WoRA","repo_kind":"listed","path":"models/aptm.py","file_url":"https://github.com/JT-Sun/Filtering-WoRA/blob/HEAD/models/aptm.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"51e7514fd736cca3"}},{"code_sha256_prefix":"38f46ef4e03fee1c","entry":"create_position_ids_from_input_ids","repo":"JT-Sun/Filtering-WoRA","repo_kind":"listed","path":"models/xroberta.py","file_url":"https://github.com/JT-Sun/Filtering-WoRA/blob/HEAD/models/xroberta.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"38f46ef4e03fee1c"}},{"code_sha256_prefix":"e7fbc7a74a3457c7","entry":"load_vocab","repo":"JT-Sun/Filtering-WoRA","repo_kind":"listed","path":"models/tokenization_bert.py","file_url":"https://github.com/JT-Sun/Filtering-WoRA/blob/HEAD/models/tokenization_bert.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e7fbc7a74a3457c7"}},{"code_sha256_prefix":"f00bedbbad13c687","entry":"read_json_file","repo":"JT-Sun/Filtering-WoRA","repo_kind":"listed","path":"cuhk_fintune_filterdelet_json.py","file_url":"https://github.com/JT-Sun/Filtering-WoRA/blob/HEAD/cuhk_fintune_filterdelet_json.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f00bedbbad13c687"}},{"code_sha256_prefix":"9e8f81565e828ebf","entry":"top_k_top_p_filtering","repo":"JT-Sun/Filtering-WoRA","repo_kind":"listed","path":"models/bert.py","file_url":"https://github.com/JT-Sun/Filtering-WoRA/blob/HEAD/models/bert.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9e8f81565e828ebf"}},{"code_sha256_prefix":"61bf152e6a42a184","entry":"window_reverse","repo":"JT-Sun/Filtering-WoRA","repo_kind":"listed","path":"models/swin_transformer.py","file_url":"https://github.com/JT-Sun/Filtering-WoRA/blob/HEAD/models/swin_transformer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"61bf152e6a42a184"}},{"code_sha256_prefix":"199b2766250bd139","entry":"interpolate_relative_pos_embed","repo":"JT-Sun/Filtering-WoRA","repo_kind":"listed","path":"models/swin_transformer.py","file_url":"https://github.com/JT-Sun/Filtering-WoRA/blob/HEAD/models/swin_transformer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"199b2766250bd139"}},{"code_sha256_prefix":"26be70dca3249c0b","entry":"load_tf_weights_in_bert","repo":"JT-Sun/Filtering-WoRA","repo_kind":"listed","path":"models/bert.py","file_url":"https://github.com/JT-Sun/Filtering-WoRA/blob/HEAD/models/bert.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"26be70dca3249c0b"}},{"code_sha256_prefix":"e3d6fbf640a143ee","entry":"reinit_scheduler_properties_mysched","repo":"JT-Sun/Filtering-WoRA","repo_kind":"listed","path":"Retrieval.py","file_url":"https://github.com/JT-Sun/Filtering-WoRA/blob/HEAD/Retrieval.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e3d6fbf640a143ee"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}