{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/robustness-in-both-domains-clip-needs-a","title":"Robustness in Both Domains: CLIP Needs a Robust Text Encoder","arxiv_id":"2506.03355","date":"2025-06-03","proceeding":null,"authors":["Elias Abad Rocamora","Christian Schlarmann","Naman Deep Singh","Yongtao Wu","Matthias Hein","Volkan Cevher"],"abstract":"Adversarial input attacks can cause a significant shift of CLIP embeddings. This can affect the downstream robustness of models incorporating CLIP in the pipeline, such as text-to-image generative models or large vision language models. While some efforts have been done towards making the CLIP image encoders robust, the robustness of text encoders remains unexplored. In this work, we cover this gap in the literature. We propose LEAF: an efficient adversarial finetuning method for the text domain, with the ability to scale to large CLIP models. Our models significantly improve the zero-shot adversarial accuracy in the text domain, while maintaining the vision performance provided by robust image encoders. When combined with text-to-image diffusion models, we can improve the generation quality under adversarial noise. When employing our robust CLIP encoders in multimodal retrieval tasks, we improve the recall under adversarial noise over standard CLIP models. Finally, we show that robust text encoders facilitate better reconstruction of input text from its embedding via direct optimization.","url_abs":"https://arxiv.org/abs/2506.03355v1","url_pdf":"https://arxiv.org/pdf/2506.03355v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[],"tasks":[],"methods":[{"method_slug":"clip","method_name":"CLIP"},{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2506.03355","atlas_url":"https://app.syntology.ai/?focus=2506.03355","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2506.03355"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/LIONS-EPFL/LEAF","reach":null}],"summary":{"ran":1,"ran_fixture":4,"ran_draft_wrong":4,"unverified":3},"by_repo_kind":{"found_in_text":{"samples":11,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":12,"samples":[{"code_sha256_prefix":"04f1b42c2632778c","entry":"ComputeLossWrapper","repo":"LIONS-EPFL/LEAF","repo_kind":"found_in_text","path":"src/robust_vlm/train/adversarial_training_clip.py","file_url":"https://github.com/LIONS-EPFL/LEAF/blob/HEAD/src/robust_vlm/train/adversarial_training_clip.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"04f1b42c2632778c"}},{"code_sha256_prefix":"f06d36585e5d6dec","entry":"ce","repo":"LIONS-EPFL/LEAF","repo_kind":"found_in_text","path":"src/robust_vlm/train/adversarial_training_clip.py","file_url":"https://github.com/LIONS-EPFL/LEAF/blob/HEAD/src/robust_vlm/train/adversarial_training_clip.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f06d36585e5d6dec"}},{"code_sha256_prefix":"1f2be2216dc3f7cd","entry":"compute_acc","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"1f2be2216dc3f7cd"}},{"code_sha256_prefix":"3bdfc4201b35c55b","entry":"compute_loss","repo":"LIONS-EPFL/LEAF","repo_kind":"found_in_text","path":"src/robust_vlm/train/adversarial_training_clip.py","file_url":"https://github.com/LIONS-EPFL/LEAF/blob/HEAD/src/robust_vlm/train/adversarial_training_clip.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3bdfc4201b35c55b"}},{"code_sha256_prefix":"2a8adc0b6b7fe4f7","entry":"l1","repo":"LIONS-EPFL/LEAF","repo_kind":"found_in_text","path":"src/robust_vlm/train/adversarial_training_clip.py","file_url":"https://github.com/LIONS-EPFL/LEAF/blob/HEAD/src/robust_vlm/train/adversarial_training_clip.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2a8adc0b6b7fe4f7"}},{"code_sha256_prefix":"caea55e9eda399d6","entry":"l2","repo":"LIONS-EPFL/LEAF","repo_kind":"found_in_text","path":"src/robust_vlm/train/adversarial_training_clip.py","file_url":"https://github.com/LIONS-EPFL/LEAF/blob/HEAD/src/robust_vlm/train/adversarial_training_clip.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"caea55e9eda399d6"}},{"code_sha256_prefix":"6fffb629c7f9b304","entry":"normalize_grad","repo":"LIONS-EPFL/LEAF","repo_kind":"found_in_text","path":"src/robust_vlm/train/adversarial_training_clip.py","file_url":"https://github.com/LIONS-EPFL/LEAF/blob/HEAD/src/robust_vlm/train/adversarial_training_clip.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6fffb629c7f9b304"}},{"code_sha256_prefix":"f9a6cd97c45d4f91","entry":"pgd","repo":"LIONS-EPFL/LEAF","repo_kind":"found_in_text","path":"src/robust_vlm/train/adversarial_training_clip.py","file_url":"https://github.com/LIONS-EPFL/LEAF/blob/HEAD/src/robust_vlm/train/adversarial_training_clip.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f9a6cd97c45d4f91"}},{"code_sha256_prefix":"74d23c944bf73c1e","entry":"project_perturbation","repo":"LIONS-EPFL/LEAF","repo_kind":"found_in_text","path":"src/robust_vlm/train/adversarial_training_clip.py","file_url":"https://github.com/LIONS-EPFL/LEAF/blob/HEAD/src/robust_vlm/train/adversarial_training_clip.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"74d23c944bf73c1e"}},{"code_sha256_prefix":"c5a17635441cc52d","entry":"evaluate","repo":"LIONS-EPFL/LEAF","repo_kind":"found_in_text","path":"src/robust_vlm/train/adversarial_training_clip.py","file_url":"https://github.com/LIONS-EPFL/LEAF/blob/HEAD/src/robust_vlm/train/adversarial_training_clip.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c5a17635441cc52d"}},{"code_sha256_prefix":"bca77f125ea3f056","entry":"train_one_epoch","repo":"LIONS-EPFL/LEAF","repo_kind":"found_in_text","path":"src/robust_vlm/train/adversarial_training_clip.py","file_url":"https://github.com/LIONS-EPFL/LEAF/blob/HEAD/src/robust_vlm/train/adversarial_training_clip.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bca77f125ea3f056"}},{"code_sha256_prefix":"a72cc58ae0995b23","entry":"unwrap_model","repo":"LIONS-EPFL/LEAF","repo_kind":"found_in_text","path":"src/robust_vlm/train/adversarial_training_clip.py","file_url":"https://github.com/LIONS-EPFL/LEAF/blob/HEAD/src/robust_vlm/train/adversarial_training_clip.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a72cc58ae0995b23"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}