{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/causal-llava-causal-disentanglement-for","title":"Causal-LLaVA: Causal Disentanglement for Mitigating Hallucination in Multimodal Large Language Models","arxiv_id":"2505.19474","date":"2025-05-26","proceeding":null,"authors":["Xinmiao Hu","Chun Wang","Ruihe An","ChenYu Shao","Xiaojun Ye","Sheng Zhou","Liangcheng Li"],"abstract":"Multimodal Large Language Models (MLLMs) have demonstrated strong performance in visual understanding tasks, yet they often suffer from object hallucinations--generating descriptions of objects that are inconsistent with or entirely absent from the input. This issue is closely related to dataset biases, where frequent co-occurrences of objects lead to entangled semantic representations across modalities. As a result, models may erroneously activate object representations that are commonly associated with the input but not actually present. To address this, we propose a causality-driven disentanglement framework that mitigates hallucinations through causal intervention. Our approach includes a Causal-Driven Projector in the visual pathway and a Causal Intervention Module integrated into the final transformer layer of the language model. These components work together to reduce spurious correlations caused by biased training data. Experimental results show that our method significantly reduces hallucinations while maintaining strong performance on multiple multimodal benchmarks. Visualization analyses further confirm improved separability of object representations. The code is available at: https://github.com/IgniSavium/Causal-LLaVA","url_abs":"https://arxiv.org/abs/2505.19474v1","url_pdf":"https://arxiv.org/pdf/2505.19474v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"causal-llava-causal-disentanglement-for","repo_url":"https://github.com/ignisavium/causal-llava","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"disentanglement","task_name":"Disentanglement"},{"task_slug":"hallucination","task_name":"Hallucination"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"object","task_name":"Object"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2505.19474","atlas_url":"https://app.syntology.ai/?focus=2505.19474","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.19474"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/IgniSavium/Causal-LLaVA","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ignisavium/causal-llava","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":3,"ran_draft_wrong":4,"ran_honours":1,"unverified":11},"by_repo_kind":{"official":{"samples":19,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"96f810f349fa18ec","entry":"CausalInterventionLinearProjector","repo":"ignisavium/causal-llava","repo_kind":"official","path":"confounders/get_projector_confounders.py","file_url":"https://github.com/ignisavium/causal-llava/blob/HEAD/confounders/get_projector_confounders.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"96f810f349fa18ec"}},{"code_sha256_prefix":"afc33375986733f9","entry":"CausalInterventionProjector","repo":"ignisavium/causal-llava","repo_kind":"official","path":"confounders/get_projector_confounders.py","file_url":"https://github.com/ignisavium/causal-llava/blob/HEAD/confounders/get_projector_confounders.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"afc33375986733f9"}},{"code_sha256_prefix":"b2f68c17571a1924","entry":"ConfounderCrossAttention","repo":"ignisavium/causal-llava","repo_kind":"official","path":"confounders/get_projector_confounders.py","file_url":"https://github.com/ignisavium/causal-llava/blob/HEAD/confounders/get_projector_confounders.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b2f68c17571a1924"}},{"code_sha256_prefix":"544bc3a3805f703a","entry":"build_vision_projector","repo":"ignisavium/causal-llava","repo_kind":"official","path":"llava/model/multimodal_projector/builder.py","file_url":"https://github.com/ignisavium/causal-llava/blob/HEAD/llava/model/multimodal_projector/builder.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"544bc3a3805f703a"}},{"code_sha256_prefix":"9d0bc30615da7d73","entry":"build_vision_projector","repo":"ignisavium/causal-llava","repo_kind":"official","path":"confounders/get_projector_confounders.py","file_url":"https://github.com/ignisavium/causal-llava/blob/HEAD/confounders/get_projector_confounders.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9d0bc30615da7d73"}},{"code_sha256_prefix":"1a16d865c0be9197","entry":"calculate_average_category_hidden_state","repo":"ignisavium/causal-llava","repo_kind":"official","path":"confounders/get_projector_confounders.py","file_url":"https://github.com/ignisavium/causal-llava/blob/HEAD/confounders/get_projector_confounders.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1a16d865c0be9197"}},{"code_sha256_prefix":"aa077d1ca4abcd21","entry":"load_image","repo":"ignisavium/causal-llava","repo_kind":"official","path":"confounders/get_projector_confounders.py","file_url":"https://github.com/ignisavium/causal-llava/blob/HEAD/confounders/get_projector_confounders.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"aa077d1ca4abcd21"}},{"code_sha256_prefix":"13577bab4f0169ac","entry":"load_mm_projector_weights","repo":"ignisavium/causal-llava","repo_kind":"official","path":"confounders/get_projector_confounders.py","file_url":"https://github.com/ignisavium/causal-llava/blob/HEAD/confounders/get_projector_confounders.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"13577bab4f0169ac"}},{"code_sha256_prefix":"cedb1eb9694b29ff","entry":"CLIPVisionTower","repo":"ignisavium/causal-llava","repo_kind":"official","path":"confounders/get_projector_confounders.py","file_url":"https://github.com/ignisavium/causal-llava/blob/HEAD/confounders/get_projector_confounders.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cedb1eb9694b29ff"}},{"code_sha256_prefix":"2db73c54632ddd6e","entry":"ProjectorConfig","repo":"ignisavium/causal-llava","repo_kind":"official","path":"confounders/get_projector_confounders.py","file_url":"https://github.com/ignisavium/causal-llava/blob/HEAD/confounders/get_projector_confounders.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2db73c54632ddd6e"}},{"code_sha256_prefix":"4d6834c552a7ec14","entry":"VisionTowerConfig","repo":"ignisavium/causal-llava","repo_kind":"official","path":"confounders/get_projector_confounders.py","file_url":"https://github.com/ignisavium/causal-llava/blob/HEAD/confounders/get_projector_confounders.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4d6834c552a7ec14"}},{"code_sha256_prefix":"50280faddf30baf6","entry":"build_vision_tower","repo":"ignisavium/causal-llava","repo_kind":"official","path":"confounders/get_projector_confounders.py","file_url":"https://github.com/ignisavium/causal-llava/blob/HEAD/confounders/get_projector_confounders.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"50280faddf30baf6"}},{"code_sha256_prefix":"cb6652b7fe967db1","entry":"extract_category_hidden_state","repo":"ignisavium/causal-llava","repo_kind":"official","path":"confounders/get_projector_confounders.py","file_url":"https://github.com/ignisavium/causal-llava/blob/HEAD/confounders/get_projector_confounders.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cb6652b7fe967db1"}},{"code_sha256_prefix":"c964b0e19df5a559","entry":"load_vision_components","repo":"ignisavium/causal-llava","repo_kind":"official","path":"confounders/get_projector_confounders.py","file_url":"https://github.com/ignisavium/causal-llava/blob/HEAD/confounders/get_projector_confounders.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c964b0e19df5a559"}},{"code_sha256_prefix":"4a7d0d350746fb32","entry":"main","repo":"ignisavium/causal-llava","repo_kind":"official","path":"confounders/get_projector_confounders.py","file_url":"https://github.com/ignisavium/causal-llava/blob/HEAD/confounders/get_projector_confounders.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4a7d0d350746fb32"}},{"code_sha256_prefix":"84ace2bb0fd9aaff","entry":"parse_arguments","repo":"ignisavium/causal-llava","repo_kind":"official","path":"confounders/get_projector_confounders.py","file_url":"https://github.com/ignisavium/causal-llava/blob/HEAD/confounders/get_projector_confounders.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"84ace2bb0fd9aaff"}},{"code_sha256_prefix":"d0420de5fae26e00","entry":"process_category_hidden_states","repo":"ignisavium/causal-llava","repo_kind":"official","path":"confounders/get_projector_confounders.py","file_url":"https://github.com/ignisavium/causal-llava/blob/HEAD/confounders/get_projector_confounders.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d0420de5fae26e00"}},{"code_sha256_prefix":"2c1d97cc7ca7b729","entry":"run_inference_and_process","repo":"ignisavium/causal-llava","repo_kind":"official","path":"confounders/get_projector_confounders.py","file_url":"https://github.com/ignisavium/causal-llava/blob/HEAD/confounders/get_projector_confounders.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2c1d97cc7ca7b729"}},{"code_sha256_prefix":"be5b16d33e19c008","entry":"save_category_hidden_states_as_bin","repo":"ignisavium/causal-llava","repo_kind":"official","path":"confounders/get_projector_confounders.py","file_url":"https://github.com/ignisavium/causal-llava/blob/HEAD/confounders/get_projector_confounders.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"be5b16d33e19c008"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}