{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/bert-lost-patience-won-t-be-robust-to-1","title":"BERT Lost Patience Won't Be Robust to Adversarial Slowdown","arxiv_id":"2310.19152","date":"2023-10-29","proceeding":"NeurIPS 2023 11","authors":["Zachary Coalson","Gabriel Ritter","Rakesh Bobba","Sanghyun Hong"],"abstract":"In this paper, we systematically evaluate the robustness of multi-exit language models against adversarial slowdown. To audit their robustness, we design a slowdown attack that generates natural adversarial text bypassing early-exit points. We use the resulting WAFFLE attack as a vehicle to conduct a comprehensive evaluation of three multi-exit mechanisms with the GLUE benchmark against adversarial slowdown. We then show our attack significantly reduces the computational savings provided by the three methods in both white-box and black-box settings. The more complex a mechanism is, the more vulnerable it is to adversarial slowdown. We also perform a linguistic analysis of the perturbed text inputs, identifying common perturbation patterns that our attack generates, and comparing them with standard adversarial text attacks. Moreover, we show that adversarial training is ineffective in defeating our slowdown attack, but input sanitization with a conversational model, e.g., ChatGPT, can remove perturbations effectively. This result suggests that future work is needed for developing efficient yet robust multi-exit models. Our code is available at: https://github.com/ztcoalson/WAFFLE","url_abs":"https://arxiv.org/abs/2310.19152v2","url_pdf":"https://arxiv.org/pdf/2310.19152v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"bert-lost-patience-won-t-be-robust-to-1","repo_url":"https://github.com/ztcoalson/waffle","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"adversarial-text","task_name":"Adversarial Text"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2310.19152","atlas_url":"https://app.syntology.ai/?focus=2310.19152","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.19152"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ztcoalson/waffle","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":2,"ran_draft_wrong":1,"unverified":2},"by_repo_kind":{"official":{"samples":5,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"839b9d0ef5c35d5c","entry":"avg_exit","repo":"ztcoalson/waffle","repo_kind":"official","path":"project/attack_exit.py","file_url":"https://github.com/ztcoalson/waffle/blob/HEAD/project/attack_exit.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"839b9d0ef5c35d5c"}},{"code_sha256_prefix":"63301b5cf89d7b03","entry":"entropy","repo":"ztcoalson/waffle","repo_kind":"official","path":"project/deebert/modeling_highway_bert.py","file_url":"https://github.com/ztcoalson/waffle/blob/HEAD/project/deebert/modeling_highway_bert.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"63301b5cf89d7b03"}},{"code_sha256_prefix":"3caf26b311c98658","entry":"get_tasks","repo":"ztcoalson/waffle","repo_kind":"official","path":"download_glue_data.py","file_url":"https://github.com/ztcoalson/waffle/blob/HEAD/download_glue_data.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3caf26b311c98658"}},{"code_sha256_prefix":"49a7658246241580","entry":"efficacy","repo":"ztcoalson/waffle","repo_kind":"official","path":"project/attack_exit.py","file_url":"https://github.com/ztcoalson/waffle/blob/HEAD/project/attack_exit.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"49a7658246241580"}},{"code_sha256_prefix":"a9744f188d4721d6","entry":"layer_info","repo":"ztcoalson/waffle","repo_kind":"official","path":"project/attack_univ_exit.py","file_url":"https://github.com/ztcoalson/waffle/blob/HEAD/project/attack_univ_exit.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a9744f188d4721d6"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}