{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2510-07835","title":"MetaDefense: Defending Finetuning-based Jailbreak Attack Before and During Generation","arxiv_id":"2510.07835","date":"2025-10-09","proceeding":"NeurIPS","authors":["Weisen Jiang","Sinno Jialin Pan"],"abstract":"This paper introduces MetaDefense, a novel framework for defending against finetuning-based jailbreak attacks in large language models (LLMs). We observe that existing defense mechanisms fail to generalize to harmful queries disguised by unseen attack templates, despite LLMs being capable of distinguishing disguised harmful queries in the embedding space. Based on these insights, we propose a two-stage defense approach: (i) pre-generation defense that detects harmful queries before response generation begins, and (ii) mid-generation defense that monitors partial responses during generation to prevent outputting more harmful content. Our MetaDefense trains the LLM to predict the harmfulness of both queries and partial responses using specialized prompts, enabling early termination of potentially harmful interactions. Extensive experiments across multiple LLM architectures (LLaMA-2-7B, Qwen-2.5-3B-Instruct, and LLaMA-3.2-3B-Instruct) demonstrate that MetaDefense significantly outperforms existing defense mechanisms, achieving robust defense against harmful queries with seen and unseen attack templates while maintaining competitive performance on benign tasks. Code is available at https://github.com/ws-jiang/MetaDefense.","url_abs":"https://arxiv.org/abs/2510.07835","url_pdf":"https://arxiv.org/pdf/2510.07835","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2510.07835","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2510.07835"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/ws-jiang/MetaDefense","reach":null}],"summary":{"ran":4,"unverified":2},"by_repo_kind":{"found_in_text":{"samples":6,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"957807d281f93841","entry":"get_ans_verify_token_ids","repo":"ws-jiang/MetaDefense","repo_kind":"found_in_text","path":"llm_utils/defense.py","file_url":"https://github.com/ws-jiang/MetaDefense/blob/HEAD/llm_utils/defense.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"957807d281f93841"}},{"code_sha256_prefix":"d011e43ba877ed86","entry":"get_answer_anchor_prompt","repo":"ws-jiang/MetaDefense","repo_kind":"found_in_text","path":"llm_utils/defense.py","file_url":"https://github.com/ws-jiang/MetaDefense/blob/HEAD/llm_utils/defense.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d011e43ba877ed86"}},{"code_sha256_prefix":"c339dbcc41ec6dea","entry":"get_harmful_harmless_probability","repo":"ws-jiang/MetaDefense","repo_kind":"found_in_text","path":"llm_utils/defense.py","file_url":"https://github.com/ws-jiang/MetaDefense/blob/HEAD/llm_utils/defense.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c339dbcc41ec6dea"}},{"code_sha256_prefix":"b07255eb026b995d","entry":"get_question_anchor_prompt","repo":"ws-jiang/MetaDefense","repo_kind":"found_in_text","path":"llm_utils/defense.py","file_url":"https://github.com/ws-jiang/MetaDefense/blob/HEAD/llm_utils/defense.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b07255eb026b995d"}},{"code_sha256_prefix":"c7941c3ba9d4102e","entry":"StopWordCriteria","repo":"ws-jiang/MetaDefense","repo_kind":"found_in_text","path":"llm_utils/defense.py","file_url":"https://github.com/ws-jiang/MetaDefense/blob/HEAD/llm_utils/defense.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c7941c3ba9d4102e"}},{"code_sha256_prefix":"5d77a6dc91897f58","entry":"meta_defense","repo":"ws-jiang/MetaDefense","repo_kind":"found_in_text","path":"llm_utils/defense.py","file_url":"https://github.com/ws-jiang/MetaDefense/blob/HEAD/llm_utils/defense.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5d77a6dc91897f58"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.LG","source":"arxiv_api"},"syntology_extracted_results":null}