{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/revisiting-character-level-adversarial","title":"Revisiting Character-level Adversarial Attacks for Language Models","arxiv_id":"2405.04346","date":"2024-05-07","proceeding":null,"authors":["Elias Abad Rocamora","Yongtao Wu","Fanghui Liu","Grigorios G. Chrysos","Volkan Cevher"],"abstract":"Adversarial attacks in Natural Language Processing apply perturbations in the character or token levels. Token-level attacks, gaining prominence for their use of gradient-based methods, are susceptible to altering sentence semantics, leading to invalid adversarial examples. While character-level attacks easily maintain semantics, they have received less attention as they cannot easily adopt popular gradient-based methods, and are thought to be easy to defend. Challenging these beliefs, we introduce Charmer, an efficient query-based adversarial attack capable of achieving high attack success rate (ASR) while generating highly similar adversarial examples. Our method successfully targets both small (BERT) and large (Llama 2) models. Specifically, on BERT with SST-2, Charmer improves the ASR in 4.84% points and the USE similarity in 8% points with respect to the previous art. Our implementation is available in https://github.com/LIONS-EPFL/Charmer.","url_abs":"https://arxiv.org/abs/2405.04346v2","url_pdf":"https://arxiv.org/pdf/2405.04346v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"revisiting-character-level-adversarial","repo_url":"https://github.com/lions-epfl/charmer","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"adversarial-attack","task_name":"Adversarial Attack"},{"task_slug":null,"task_name":"SST-2"},{"task_slug":"sentence","task_name":"Sentence"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"use","method_name":"USE"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2405.04346","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.04346"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/LIONS-EPFL/Charmer","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/lions-epfl/charmer","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":13,"ran_draft_wrong":9,"ran_fixture":1,"unverified":8},"by_repo_kind":{"official":{"samples":31,"ran":23,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"6a768c7d89fa45a6","entry":"CrossEntropyLoss_negative","repo":"lions-epfl/charmer","repo_kind":"official","path":"charmer.py","file_url":"https://github.com/lions-epfl/charmer/blob/HEAD/charmer.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6a768c7d89fa45a6"}},{"code_sha256_prefix":"b87dfb2a13227d05","entry":"all_strings_within_edit1","repo":"lions-epfl/charmer","repo_kind":"official","path":"charmer.py","file_url":"https://github.com/lions-epfl/charmer/blob/HEAD/charmer.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b87dfb2a13227d05"}},{"code_sha256_prefix":"232efbd63c8df619","entry":"all_strings_within_editx","repo":"lions-epfl/charmer","repo_kind":"official","path":"charmer.py","file_url":"https://github.com/lions-epfl/charmer/blob/HEAD/charmer.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"232efbd63c8df619"}},{"code_sha256_prefix":"96a7b58d20a2005f","entry":"backtrace","repo":"lions-epfl/charmer","repo_kind":"official","path":"charmer.py","file_url":"https://github.com/lions-epfl/charmer/blob/HEAD/charmer.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"96a7b58d20a2005f"}},{"code_sha256_prefix":"92dfc8c506947d1c","entry":"compute_centroid_distances","repo":"LIONS-EPFL/Charmer","repo_kind":"official","path":"baseline/roben/agglom_clusters.py","file_url":"https://github.com/LIONS-EPFL/Charmer/blob/HEAD/baseline/roben/agglom_clusters.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"92dfc8c506947d1c"}},{"code_sha256_prefix":"7a4e3ab49ac3c159","entry":"concat_prompt_question_label","repo":"lions-epfl/charmer","repo_kind":"official","path":"charmer.py","file_url":"https://github.com/lions-epfl/charmer/blob/HEAD/charmer.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7a4e3ab49ac3c159"}},{"code_sha256_prefix":"4e36262a542c16fb","entry":"ctx_noparamgrad","repo":"lions-epfl/charmer","repo_kind":"official","path":"charmer.py","file_url":"https://github.com/lions-epfl/charmer/blob/HEAD/charmer.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4e36262a542c16fb"}},{"code_sha256_prefix":"7b06bf5b0955fdac","entry":"generate_all_sentences","repo":"lions-epfl/charmer","repo_kind":"official","path":"charmer.py","file_url":"https://github.com/lions-epfl/charmer/blob/HEAD/charmer.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7b06bf5b0955fdac"}},{"code_sha256_prefix":"b82a34d25dd75308","entry":"generate_all_sentences_at_z","repo":"lions-epfl/charmer","repo_kind":"official","path":"charmer.py","file_url":"https://github.com/lions-epfl/charmer/blob/HEAD/charmer.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b82a34d25dd75308"}},{"code_sha256_prefix":"7f56e8436fe9b621","entry":"generate_sentence","repo":"lions-epfl/charmer","repo_kind":"official","path":"charmer.py","file_url":"https://github.com/lions-epfl/charmer/blob/HEAD/charmer.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7f56e8436fe9b621"}},{"code_sha256_prefix":"904d63da437332d4","entry":"get_class_num","repo":"LIONS-EPFL/Charmer","repo_kind":"official","path":"dataloader.py","file_url":"https://github.com/LIONS-EPFL/Charmer/blob/HEAD/dataloader.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"904d63da437332d4"}},{"code_sha256_prefix":"55554ebb64125749","entry":"get_goals_and_targets","repo":"LIONS-EPFL/Charmer","repo_kind":"official","path":"utils_jailbreak.py","file_url":"https://github.com/LIONS-EPFL/Charmer/blob/HEAD/utils_jailbreak.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"55554ebb64125749"}},{"code_sha256_prefix":"28413f6ed2f091a9","entry":"get_module_training_state","repo":"LIONS-EPFL/Charmer","repo_kind":"official","path":"context.py","file_url":"https://github.com/LIONS-EPFL/Charmer/blob/HEAD/context.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"28413f6ed2f091a9"}},{"code_sha256_prefix":"41eb44fa1a771cd1","entry":"get_param_grad_state","repo":"lions-epfl/charmer","repo_kind":"official","path":"charmer.py","file_url":"https://github.com/lions-epfl/charmer/blob/HEAD/charmer.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"41eb44fa1a771cd1"}},{"code_sha256_prefix":"75df2d0c0186e517","entry":"get_task_type","repo":"LIONS-EPFL/Charmer","repo_kind":"official","path":"dataloader.py","file_url":"https://github.com/LIONS-EPFL/Charmer/blob/HEAD/dataloader.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"75df2d0c0186e517"}},{"code_sha256_prefix":"593614c84d8b37c2","entry":"get_vocabulary","repo":"LIONS-EPFL/Charmer","repo_kind":"official","path":"utils.py","file_url":"https://github.com/LIONS-EPFL/Charmer/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"593614c84d8b37c2"}},{"code_sha256_prefix":"456e7d855a79eb1e","entry":"llm_input_processor","repo":"lions-epfl/charmer","repo_kind":"official","path":"charmer.py","file_url":"https://github.com/lions-epfl/charmer/blob/HEAD/charmer.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"456e7d855a79eb1e"}},{"code_sha256_prefix":"28879c448358c558","entry":"margin_loss_llm","repo":"lions-epfl/charmer","repo_kind":"official","path":"charmer.py","file_url":"https://github.com/lions-epfl/charmer/blob/HEAD/charmer.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"28879c448358c558"}},{"code_sha256_prefix":"7eae0eca72c38f98","entry":"margin_loss_lm","repo":"lions-epfl/charmer","repo_kind":"official","path":"charmer.py","file_url":"https://github.com/lions-epfl/charmer/blob/HEAD/charmer.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7eae0eca72c38f98"}},{"code_sha256_prefix":"f38056701ce7c0b6","entry":"margin_loss_lm_batched","repo":"lions-epfl/charmer","repo_kind":"official","path":"charmer.py","file_url":"https://github.com/lions-epfl/charmer/blob/HEAD/charmer.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f38056701ce7c0b6"}},{"code_sha256_prefix":"11a3348eb0e4ba60","entry":"target_loss","repo":"LIONS-EPFL/Charmer","repo_kind":"official","path":"utils_jailbreak.py","file_url":"https://github.com/LIONS-EPFL/Charmer/blob/HEAD/utils_jailbreak.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"11a3348eb0e4ba60"}},{"code_sha256_prefix":"12efa3ddd30cf7c2","entry":"update_mu","repo":"LIONS-EPFL/Charmer","repo_kind":"official","path":"baseline/roben/agglom_clusters.py","file_url":"https://github.com/LIONS-EPFL/Charmer/blob/HEAD/baseline/roben/agglom_clusters.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"12efa3ddd30cf7c2"}},{"code_sha256_prefix":"f0c1289fd2565267","entry":"word_edit_distance","repo":"lions-epfl/charmer","repo_kind":"official","path":"charmer.py","file_url":"https://github.com/lions-epfl/charmer/blob/HEAD/charmer.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f0c1289fd2565267"}},{"code_sha256_prefix":"59ab5813bed83cb2","entry":"Charmer","repo":"lions-epfl/charmer","repo_kind":"official","path":"charmer.py","file_url":"https://github.com/lions-epfl/charmer/blob/HEAD/charmer.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"59ab5813bed83cb2"}},{"code_sha256_prefix":"a299d58465c7bf59","entry":"combined_emb_input","repo":"LIONS-EPFL/Charmer","repo_kind":"official","path":"utils.py","file_url":"https://github.com/LIONS-EPFL/Charmer/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a299d58465c7bf59"}},{"code_sha256_prefix":"8ae9f9ad369707a9","entry":"get_llm_embeddings","repo":"lions-epfl/charmer","repo_kind":"official","path":"charmer.py","file_url":"https://github.com/lions-epfl/charmer/blob/HEAD/charmer.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8ae9f9ad369707a9"}},{"code_sha256_prefix":"6297161e122df004","entry":"get_llm_encoder","repo":"lions-epfl/charmer","repo_kind":"official","path":"charmer.py","file_url":"https://github.com/lions-epfl/charmer/blob/HEAD/charmer.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6297161e122df004"}},{"code_sha256_prefix":"7f504481b40a6171","entry":"load_attack_dataset","repo":"LIONS-EPFL/Charmer","repo_kind":"official","path":"dataloader.py","file_url":"https://github.com/LIONS-EPFL/Charmer/blob/HEAD/dataloader.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7f504481b40a6171"}},{"code_sha256_prefix":"cb53a78f621babdb","entry":"resize_embeddings","repo":"lions-epfl/charmer","repo_kind":"official","path":"charmer.py","file_url":"https://github.com/lions-epfl/charmer/blob/HEAD/charmer.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cb53a78f621babdb"}},{"code_sha256_prefix":"e75d10fb3cfbb60c","entry":"set_param_grad_off","repo":"lions-epfl/charmer","repo_kind":"official","path":"charmer.py","file_url":"https://github.com/lions-epfl/charmer/blob/HEAD/charmer.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e75d10fb3cfbb60c"}},{"code_sha256_prefix":"ea9f4cd8cc668205","entry":"set_param_grad_state","repo":"lions-epfl/charmer","repo_kind":"official","path":"charmer.py","file_url":"https://github.com/lions-epfl/charmer/blob/HEAD/charmer.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ea9f4cd8cc668205"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}