{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/from-uncertainty-to-trust-enhancing","title":"From Uncertainty to Trust: Enhancing Reliability in Vision-Language Models with Uncertainty-Guided Dropout Decoding","arxiv_id":"2412.06474","date":"2024-12-09","proceeding":null,"authors":["Yixiong Fang","Ziran Yang","Zhaorun Chen","Zhuokai Zhao","Jiawei Zhou"],"abstract":"Large vision-language models (LVLMs) demonstrate remarkable capabilities in multimodal tasks but are prone to misinterpreting visual inputs, often resulting in hallucinations and unreliable outputs. To address these challenges, we propose Dropout Decoding, a novel inference-time approach that quantifies the uncertainty of visual tokens and selectively masks uncertain tokens to improve decoding. Our method measures the uncertainty of each visual token by projecting it onto the text space and decomposing it into aleatoric and epistemic components. Specifically, we focus on epistemic uncertainty, which captures perception-related errors more effectively. Inspired by dropout regularization, we introduce uncertainty-guided token dropout, which applies the dropout principle to input visual tokens instead of model parameters, and during inference rather than training. By aggregating predictions from an ensemble of masked decoding contexts, Dropout Decoding robustly mitigates errors arising from visual token misinterpretations. Evaluations on benchmarks including CHAIR, THRONE, and MMBench demonstrate that Dropout Decoding significantly reduces object hallucinations (OH) and enhances both reliability and quality of LVLM outputs across diverse visual contexts.","url_abs":"https://arxiv.org/abs/2412.06474v1","url_pdf":"https://arxiv.org/pdf/2412.06474v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"from-uncertainty-to-trust-enhancing","repo_url":"https://github.com/kigb/dropoutdecoding","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[],"methods":[{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"focus","method_name":"Focus"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2412.06474","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2412.06474"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/kigb/DropoutDecoding","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kigb/dropoutdecoding","reach":{"status":"ok"}}],"summary":{"ran_draft_wrong":3,"unverified":15},"by_repo_kind":{"official":{"samples":18,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":18,"samples":[{"code_sha256_prefix":"25585704d77cafb4","entry":"combine_coco_captions","repo":"kigb/DropoutDecoding","repo_kind":"official","path":"chair_test/chair_metrics/chair.py","file_url":"https://github.com/kigb/DropoutDecoding/blob/HEAD/chair_test/chair_metrics/chair.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"25585704d77cafb4"}},{"code_sha256_prefix":"66415dc1dd50f555","entry":"combine_coco_instances","repo":"kigb/DropoutDecoding","repo_kind":"official","path":"chair_test/chair_metrics/chair.py","file_url":"https://github.com/kigb/DropoutDecoding/blob/HEAD/chair_test/chair_metrics/chair.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"66415dc1dd50f555"}},{"code_sha256_prefix":"53d425eb1278ddda","entry":"load_json","repo":"kigb/DropoutDecoding","repo_kind":"official","path":"utils/compare.py","file_url":"https://github.com/kigb/DropoutDecoding/blob/HEAD/utils/compare.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"53d425eb1278ddda"}},{"code_sha256_prefix":"fe66de066c32d0fc","entry":"add_diffusion_noise","repo":"kigb/DropoutDecoding","repo_kind":"official","path":"models/VCD/vcd_add_noise.py","file_url":"https://github.com/kigb/DropoutDecoding/blob/HEAD/models/VCD/vcd_add_noise.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fe66de066c32d0fc"}},{"code_sha256_prefix":"eb627f04ba74cba6","entry":"argmax","repo":"kigb/DropoutDecoding","repo_kind":"official","path":"models/llavanext.py","file_url":"https://github.com/kigb/DropoutDecoding/blob/HEAD/models/llavanext.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"eb627f04ba74cba6"}},{"code_sha256_prefix":"a24f296c7f2750a0","entry":"get_blank_prediction_path","repo":"kigb/DropoutDecoding","repo_kind":"official","path":"chair_test/chair_metrics/lm_consistency.py","file_url":"https://github.com/kigb/DropoutDecoding/blob/HEAD/chair_test/chair_metrics/lm_consistency.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a24f296c7f2750a0"}},{"code_sha256_prefix":"d9a1b1be427abb03","entry":"get_consistency","repo":"kigb/DropoutDecoding","repo_kind":"official","path":"chair_test/chair_metrics/misc.py","file_url":"https://github.com/kigb/DropoutDecoding/blob/HEAD/chair_test/chair_metrics/misc.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d9a1b1be427abb03"}},{"code_sha256_prefix":"c9a73d69abdd252a","entry":"get_im_consistency","repo":"kigb/DropoutDecoding","repo_kind":"official","path":"chair_test/chair_metrics/im_consistency.py","file_url":"https://github.com/kigb/DropoutDecoding/blob/HEAD/chair_test/chair_metrics/im_consistency.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c9a73d69abdd252a"}},{"code_sha256_prefix":"1b18bf83296c81a8","entry":"get_label_dicts","repo":"kigb/DropoutDecoding","repo_kind":"official","path":"chair_test/chair_metrics/im_consistency.py","file_url":"https://github.com/kigb/DropoutDecoding/blob/HEAD/chair_test/chair_metrics/im_consistency.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1b18bf83296c81a8"}},{"code_sha256_prefix":"521fb486761a4172","entry":"get_sentence_scores_from_hallucination_file","repo":"kigb/DropoutDecoding","repo_kind":"official","path":"chair_test/chair_metrics/misc.py","file_url":"https://github.com/kigb/DropoutDecoding/blob/HEAD/chair_test/chair_metrics/misc.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"521fb486761a4172"}},{"code_sha256_prefix":"0bb2534b6701d132","entry":"hallucination_file_to_dict","repo":"kigb/DropoutDecoding","repo_kind":"official","path":"chair_test/chair_metrics/misc.py","file_url":"https://github.com/kigb/DropoutDecoding/blob/HEAD/chair_test/chair_metrics/misc.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0bb2534b6701d132"}},{"code_sha256_prefix":"c6889eebaff1566f","entry":"load_generated_captions","repo":"kigb/DropoutDecoding","repo_kind":"official","path":"chair_test/chair_metrics/chair.py","file_url":"https://github.com/kigb/DropoutDecoding/blob/HEAD/chair_test/chair_metrics/chair.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c6889eebaff1566f"}},{"code_sha256_prefix":"3fda39b99e8fcada","entry":"read_vocab","repo":"kigb/DropoutDecoding","repo_kind":"official","path":"chair_test/chair_metrics/lm_consistency.py","file_url":"https://github.com/kigb/DropoutDecoding/blob/HEAD/chair_test/chair_metrics/lm_consistency.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3fda39b99e8fcada"}},{"code_sha256_prefix":"265b8739e27fe5c8","entry":"sample","repo":"kigb/dropoutdecoding","repo_kind":"official","path":"models/VCD/vcd_sample.py","file_url":"https://github.com/kigb/dropoutdecoding/blob/HEAD/models/VCD/vcd_sample.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"265b8739e27fe5c8"}},{"code_sha256_prefix":"64b4a235a9c34b8a","entry":"select_by_average","repo":"kigb/DropoutDecoding","repo_kind":"official","path":"models/llava.py","file_url":"https://github.com/kigb/DropoutDecoding/blob/HEAD/models/llava.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"64b4a235a9c34b8a"}},{"code_sha256_prefix":"2a5e72bfe7b8597f","entry":"select_by_vote","repo":"kigb/DropoutDecoding","repo_kind":"official","path":"models/llava.py","file_url":"https://github.com/kigb/DropoutDecoding/blob/HEAD/models/llava.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2a5e72bfe7b8597f"}},{"code_sha256_prefix":"13246077f0242fbf","entry":"select_by_vote","repo":"kigb/DropoutDecoding","repo_kind":"official","path":"models/llavanext.py","file_url":"https://github.com/kigb/DropoutDecoding/blob/HEAD/models/llavanext.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"13246077f0242fbf"}},{"code_sha256_prefix":"bfe1425c081169ef","entry":"softmax","repo":"kigb/DropoutDecoding","repo_kind":"official","path":"chair_test/chair_metrics/lm_consistency.py","file_url":"https://github.com/kigb/DropoutDecoding/blob/HEAD/chair_test/chair_metrics/lm_consistency.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bfe1425c081169ef"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}