{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/unlearning-sensitive-information-in","title":"Unlearning Sensitive Information in Multimodal LLMs: Benchmark and Attack-Defense Evaluation","arxiv_id":"2505.01456","date":"2025-05-01","proceeding":null,"authors":["Vaidehi Patil","Yi-Lin Sung","Peter Hase","Jie Peng","Tianlong Chen","Mohit Bansal"],"abstract":"LLMs trained on massive datasets may inadvertently acquire sensitive information such as personal details and potentially harmful content. This risk is further heightened in multimodal LLMs as they integrate information from multiple modalities (image and text). Adversaries can exploit this knowledge through multimodal prompts to extract sensitive details. Evaluating how effectively MLLMs can forget such information (targeted unlearning) necessitates the creation of high-quality, well-annotated image-text pairs. While prior work on unlearning has focused on text, multimodal unlearning remains underexplored. To address this gap, we first introduce a multimodal unlearning benchmark, UnLOK-VQA (Unlearning Outside Knowledge VQA), as well as an attack-and-defense framework to evaluate methods for deleting specific multimodal knowledge from MLLMs. We extend a visual question-answering dataset using an automated pipeline that generates varying-proximity samples for testing generalization and specificity, followed by manual filtering for maintaining high quality. We then evaluate six defense objectives against seven attacks (four whitebox, three blackbox), including a novel whitebox method leveraging interpretability of hidden states. Our results show multimodal attacks outperform text- or image-only ones, and that the most effective defense removes answer information from internal model states. Additionally, larger models exhibit greater post-editing robustness, suggesting that scale enhances safety. UnLOK-VQA provides a rigorous benchmark for advancing unlearning in MLLMs.","url_abs":"https://arxiv.org/abs/2505.01456v1","url_pdf":"https://arxiv.org/pdf/2505.01456v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"unlearning-sensitive-information-in","repo_url":"https://github.com/vaidehi99/unlok-vqa","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"specificity","task_name":"Specificity"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"},{"task_slug":"visual-question-answering","task_name":"Visual Question Answering (VQA)"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2505.01456","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.01456"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/vaidehi99/unlok-vqa","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":3,"ran":2,"ran_violates":1,"ran_honours":1,"unverified":1},"by_repo_kind":{"official":{"samples":8,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"42a46570620cd9fa","entry":"get_chunk","repo":"vaidehi99/unlok-vqa","repo_kind":"official","path":"LLaVA/llava/eval/model_vqa.py","file_url":"https://github.com/vaidehi99/unlok-vqa/blob/HEAD/LLaVA/llava/eval/model_vqa.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"42a46570620cd9fa"}},{"code_sha256_prefix":"920b394e7ad80c53","entry":"hierarchical_subsequence","repo":"vaidehi99/unlok-vqa","repo_kind":"official","path":"util/nethook.py","file_url":"https://github.com/vaidehi99/unlok-vqa/blob/HEAD/util/nethook.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"920b394e7ad80c53"}},{"code_sha256_prefix":"bae18947b56f2be1","entry":"is_none","repo":"vaidehi99/unlok-vqa","repo_kind":"official","path":"LLaVA/llava/eval/model_vqa_mmbench.py","file_url":"https://github.com/vaidehi99/unlok-vqa/blob/HEAD/LLaVA/llava/eval/model_vqa_mmbench.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bae18947b56f2be1"}},{"code_sha256_prefix":"9b3c1cb391672ccb","entry":"load_image","repo":"vaidehi99/unlok-vqa","repo_kind":"official","path":"LLaVA/predict.py","file_url":"https://github.com/vaidehi99/unlok-vqa/blob/HEAD/LLaVA/predict.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9b3c1cb391672ccb"}},{"code_sha256_prefix":"70f6ab8bde55420e","entry":"recursive_copy","repo":"vaidehi99/unlok-vqa","repo_kind":"official","path":"util/nethook.py","file_url":"https://github.com/vaidehi99/unlok-vqa/blob/HEAD/util/nethook.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"70f6ab8bde55420e"}},{"code_sha256_prefix":"076c252c52cbb161","entry":"split_list","repo":"vaidehi99/unlok-vqa","repo_kind":"official","path":"LLaVA/llava/eval/model_vqa.py","file_url":"https://github.com/vaidehi99/unlok-vqa/blob/HEAD/LLaVA/llava/eval/model_vqa.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"076c252c52cbb161"}},{"code_sha256_prefix":"440ff98c2b1ae1aa","entry":"subsequence","repo":"vaidehi99/unlok-vqa","repo_kind":"official","path":"util/nethook.py","file_url":"https://github.com/vaidehi99/unlok-vqa/blob/HEAD/util/nethook.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"440ff98c2b1ae1aa"}},{"code_sha256_prefix":"25fe426deb51e58f","entry":"create_qg_prompt","repo":"vaidehi99/unlok-vqa","repo_kind":"official","path":"LLaVA/qg.py","file_url":"https://github.com/vaidehi99/unlok-vqa/blob/HEAD/LLaVA/qg.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"25fe426deb51e58f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}