{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/attacking-multimodal-os-agents-with-malicious","title":"Attacking Multimodal OS Agents with Malicious Image Patches","arxiv_id":"2503.10809","date":"2025-03-13","proceeding":null,"authors":["Lukas Aichberger","Alasdair Paren","Yarin Gal","Philip Torr","Adel Bibi"],"abstract":"Recent advances in operating system (OS) agents enable vision-language models to interact directly with the graphical user interface of an OS. These multimodal OS agents autonomously perform computer-based tasks in response to a single prompt via application programming interfaces (APIs). Such APIs typically support low-level operations, including mouse clicks, keyboard inputs, and screenshot captures. We introduce a novel attack vector: malicious image patches (MIPs) that have been adversarially perturbed so that, when captured in a screenshot, they cause an OS agent to perform harmful actions by exploiting specific APIs. For instance, MIPs embedded in desktop backgrounds or shared on social media can redirect an agent to a malicious website, enabling further exploitation. These MIPs generalise across different user requests and screen layouts, and remain effective for multiple OS agents. The existence of such attacks highlights critical security vulnerabilities in OS agents, which should be carefully addressed before their widespread adoption.","url_abs":"https://arxiv.org/abs/2503.10809v1","url_pdf":"https://arxiv.org/pdf/2503.10809v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2503.10809","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2503.10809"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/AIchberger/mip-against-agent","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"unverified":22},"by_repo_kind":{"found_in_text":{"samples":22,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"4a1f4e792a6ed7cd","entry":"append_results","repo":"AIchberger/mip-against-agent","repo_kind":"found_in_text","path":"src/mip_attack/reporting.py","file_url":"https://github.com/AIchberger/mip-against-agent/blob/HEAD/src/mip_attack/reporting.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4a1f4e792a6ed7cd"}},{"code_sha256_prefix":"695a398a0fdd89e9","entry":"backward_model_loss","repo":"AIchberger/mip-against-agent","repo_kind":"found_in_text","path":"src/mip_attack/optimization.py","file_url":"https://github.com/AIchberger/mip-against-agent/blob/HEAD/src/mip_attack/optimization.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"695a398a0fdd89e9"}},{"code_sha256_prefix":"b9361dc2dda419da","entry":"best_patch_rank","repo":"AIchberger/mip-against-agent","repo_kind":"found_in_text","path":"src/mip_attack/optimization.py","file_url":"https://github.com/AIchberger/mip-against-agent/blob/HEAD/src/mip_attack/optimization.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b9361dc2dda419da"}},{"code_sha256_prefix":"7f1dd1b398ed9382","entry":"build_user_msg_visual","repo":"AIchberger/mip-against-agent","repo_kind":"found_in_text","path":"src/mip_attack/prompts.py","file_url":"https://github.com/AIchberger/mip-against-agent/blob/HEAD/src/mip_attack/prompts.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7f1dd1b398ed9382"}},{"code_sha256_prefix":"d73031427f43ab3b","entry":"centered_patch_ranges","repo":"AIchberger/mip-against-agent","repo_kind":"found_in_text","path":"src/mip_attack/config.py","file_url":"https://github.com/AIchberger/mip-against-agent/blob/HEAD/src/mip_attack/config.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d73031427f43ab3b"}},{"code_sha256_prefix":"6ea9ca43cd05ac45","entry":"ensure_results_file","repo":"AIchberger/mip-against-agent","repo_kind":"found_in_text","path":"src/mip_attack/reporting.py","file_url":"https://github.com/AIchberger/mip-against-agent/blob/HEAD/src/mip_attack/reporting.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6ea9ca43cd05ac45"}},{"code_sha256_prefix":"d690aada5f8669d2","entry":"filter_entities_with_mask","repo":"AIchberger/mip-against-agent","repo_kind":"found_in_text","path":"src/mip_attack/a11y.py","file_url":"https://github.com/AIchberger/mip-against-agent/blob/HEAD/src/mip_attack/a11y.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d690aada5f8669d2"}},{"code_sha256_prefix":"7cbc0ee1e93d5dec","entry":"filter_nonvis_and_oob_entities","repo":"AIchberger/mip-against-agent","repo_kind":"found_in_text","path":"src/mip_attack/a11y.py","file_url":"https://github.com/AIchberger/mip-against-agent/blob/HEAD/src/mip_attack/a11y.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7cbc0ee1e93d5dec"}},{"code_sha256_prefix":"023a841443ebee6c","entry":"get_mask_from_entities","repo":"AIchberger/mip-against-agent","repo_kind":"found_in_text","path":"src/mip_attack/a11y.py","file_url":"https://github.com/AIchberger/mip-against-agent/blob/HEAD/src/mip_attack/a11y.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"023a841443ebee6c"}},{"code_sha256_prefix":"0d40f28ae88b0be6","entry":"normalize_image","repo":"AIchberger/mip-against-agent","repo_kind":"found_in_text","path":"src/mip_attack/model.py","file_url":"https://github.com/AIchberger/mip-against-agent/blob/HEAD/src/mip_attack/model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0d40f28ae88b0be6"}},{"code_sha256_prefix":"fa74fb20ad5c6635","entry":"paper_conditions","repo":"AIchberger/mip-against-agent","repo_kind":"found_in_text","path":"src/mip_attack/protocol.py","file_url":"https://github.com/AIchberger/mip-against-agent/blob/HEAD/src/mip_attack/protocol.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fa74fb20ad5c6635"}},{"code_sha256_prefix":"b8f593e790ff6e89","entry":"place_png_patch","repo":"AIchberger/mip-against-agent","repo_kind":"found_in_text","path":"src/mip_attack/agent.py","file_url":"https://github.com/AIchberger/mip-against-agent/blob/HEAD/src/mip_attack/agent.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b8f593e790ff6e89"}},{"code_sha256_prefix":"8df7273a1174a5b9","entry":"previous_actions_text","repo":"AIchberger/mip-against-agent","repo_kind":"found_in_text","path":"src/mip_attack/agent.py","file_url":"https://github.com/AIchberger/mip-against-agent/blob/HEAD/src/mip_attack/agent.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8df7273a1174a5b9"}},{"code_sha256_prefix":"812af51bcfcfea5c","entry":"read_target","repo":"AIchberger/mip-against-agent","repo_kind":"found_in_text","path":"src/mip_attack/config.py","file_url":"https://github.com/AIchberger/mip-against-agent/blob/HEAD/src/mip_attack/config.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"812af51bcfcfea5c"}},{"code_sha256_prefix":"a2421fdbc5dc7073","entry":"relative_artifact_path","repo":"AIchberger/mip-against-agent","repo_kind":"found_in_text","path":"src/mip_attack/artifacts.py","file_url":"https://github.com/AIchberger/mip-against-agent/blob/HEAD/src/mip_attack/artifacts.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a2421fdbc5dc7073"}},{"code_sha256_prefix":"ff0604a9ed196593","entry":"resolve_path","repo":"AIchberger/mip-against-agent","repo_kind":"found_in_text","path":"src/mip_attack/config.py","file_url":"https://github.com/AIchberger/mip-against-agent/blob/HEAD/src/mip_attack/config.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ff0604a9ed196593"}},{"code_sha256_prefix":"6d4d608b0d69807f","entry":"retain_best_patch","repo":"AIchberger/mip-against-agent","repo_kind":"found_in_text","path":"src/mip_attack/optimization.py","file_url":"https://github.com/AIchberger/mip-against-agent/blob/HEAD/src/mip_attack/optimization.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6d4d608b0d69807f"}},{"code_sha256_prefix":"00eed04c6bc6f9c9","entry":"select_targeted_pair","repo":"AIchberger/mip-against-agent","repo_kind":"found_in_text","path":"src/mip_attack/protocol.py","file_url":"https://github.com/AIchberger/mip-against-agent/blob/HEAD/src/mip_attack/protocol.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"00eed04c6bc6f9c9"}},{"code_sha256_prefix":"237d78c2b5521450","entry":"tile_image","repo":"AIchberger/mip-against-agent","repo_kind":"found_in_text","path":"src/mip_attack/model.py","file_url":"https://github.com/AIchberger/mip-against-agent/blob/HEAD/src/mip_attack/model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"237d78c2b5521450"}},{"code_sha256_prefix":"4cf99e43d4302ec0","entry":"torch_load","repo":"AIchberger/mip-against-agent","repo_kind":"found_in_text","path":"src/mip_attack/artifacts.py","file_url":"https://github.com/AIchberger/mip-against-agent/blob/HEAD/src/mip_attack/artifacts.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4cf99e43d4302ec0"}},{"code_sha256_prefix":"be561a8361863fa7","entry":"unnormalize_image","repo":"AIchberger/mip-against-agent","repo_kind":"found_in_text","path":"src/mip_attack/model.py","file_url":"https://github.com/AIchberger/mip-against-agent/blob/HEAD/src/mip_attack/model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"be561a8361863fa7"}},{"code_sha256_prefix":"5b1a5a43a05c36df","entry":"validate_id","repo":"AIchberger/mip-against-agent","repo_kind":"found_in_text","path":"src/mip_attack/artifacts.py","file_url":"https://github.com/AIchberger/mip-against-agent/blob/HEAD/src/mip_attack/artifacts.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5b1a5a43a05c36df"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}