{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/jailbreaking-as-a-reward-misspecification","title":"Jailbreaking as a Reward Misspecification Problem","arxiv_id":"2406.14393","date":"2024-06-20","proceeding":null,"authors":["Zhihui Xie","Jiahui Gao","Lei LI","Zhenguo Li","Qi Liu","Lingpeng Kong"],"abstract":"The widespread adoption of large language models (LLMs) has raised concerns about their safety and reliability, particularly regarding their vulnerability to adversarial attacks. In this paper, we propose a novel perspective that attributes this vulnerability to reward misspecification during the alignment process. This misspecification occurs when the reward function fails to accurately capture the intended behavior, leading to misaligned model outputs. We introduce a metric ReGap to quantify the extent of reward misspecification and demonstrate its effectiveness and robustness in detecting harmful backdoor prompts. Building upon these insights, we present ReMiss, a system for automated red teaming that generates adversarial prompts in a reward-misspecified space. ReMiss achieves state-of-the-art attack success rates on the AdvBench benchmark against various target aligned LLMs while preserving the human readability of the generated prompts. Furthermore, these attacks on open-source models demonstrate high transferability to closed-source models like GPT-4o and out-of-distribution tasks from HarmBench. Detailed analysis highlights the unique advantages of the proposed reward misspecification objective compared to previous methods, offering new insights for improving LLM safety and robustness.","url_abs":"https://arxiv.org/abs/2406.14393v5","url_pdf":"https://arxiv.org/pdf/2406.14393v5.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"jailbreaking-as-a-reward-misspecification","repo_url":"https://github.com/zhxieml/remiss-jailbreak","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"red-teaming","task_name":"Red Teaming"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2406.14393","atlas_url":"https://app.syntology.ai/?focus=2406.14393","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.14393"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/zhxieml/remiss-jailbreak","reach":null}],"summary":{"ran":2,"ran_draft_wrong":2,"ran_fixture":1,"ran_honours":2,"unverified":8},"by_repo_kind":{"official":{"samples":15,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":15,"samples":[{"code_sha256_prefix":"f7e6ed8ae82ad71f","entry":"EmptySeq","repo":"zhxieml/remiss-jailbreak","repo_kind":"official","path":"src/remissopt.py","file_url":"https://github.com/zhxieml/remiss-jailbreak/blob/HEAD/src/remissopt.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f7e6ed8ae82ad71f"}},{"code_sha256_prefix":"b33c78601d890827","entry":"Seq","repo":"zhxieml/remiss-jailbreak","repo_kind":"official","path":"src/remissopt.py","file_url":"https://github.com/zhxieml/remiss-jailbreak/blob/HEAD/src/remissopt.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b33c78601d890827"}},{"code_sha256_prefix":"421d4ba5e59039da","entry":"add_dummy_dim_to_slice","repo":"zhxieml/remiss-jailbreak","repo_kind":"official","path":"src/remissopt.py","file_url":"https://github.com/zhxieml/remiss-jailbreak/blob/HEAD/src/remissopt.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"421d4ba5e59039da"}},{"code_sha256_prefix":"298878ca905883f4","entry":"apply_repetition_penalty","repo":"zhxieml/remiss-jailbreak","repo_kind":"official","path":"src/remissopt.py","file_url":"https://github.com/zhxieml/remiss-jailbreak/blob/HEAD/src/remissopt.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"298878ca905883f4"}},{"code_sha256_prefix":"a70d14fb2791dd29","entry":"expand_for_broadcast_list","repo":"zhxieml/remiss-jailbreak","repo_kind":"official","path":"src/remissopt.py","file_url":"https://github.com/zhxieml/remiss-jailbreak/blob/HEAD/src/remissopt.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a70d14fb2791dd29"}},{"code_sha256_prefix":"249485c3fcfb764e","entry":"expand_for_broadcast_tensor","repo":"zhxieml/remiss-jailbreak","repo_kind":"official","path":"src/remissopt.py","file_url":"https://github.com/zhxieml/remiss-jailbreak/blob/HEAD/src/remissopt.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"249485c3fcfb764e"}},{"code_sha256_prefix":"b5c4e0c154efef64","entry":"select_next_token_candidates","repo":"zhxieml/remiss-jailbreak","repo_kind":"official","path":"src/remissopt.py","file_url":"https://github.com/zhxieml/remiss-jailbreak/blob/HEAD/src/remissopt.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b5c4e0c154efef64"}},{"code_sha256_prefix":"b50e2d1935c06431","entry":"MergedSeq","repo":"zhxieml/remiss-jailbreak","repo_kind":"official","path":"src/remissopt.py","file_url":"https://github.com/zhxieml/remiss-jailbreak/blob/HEAD/src/remissopt.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b50e2d1935c06431"}},{"code_sha256_prefix":"bf9d280eb54736b7","entry":"calculate_reward","repo":"zhxieml/remiss-jailbreak","repo_kind":"official","path":"src/remissopt.py","file_url":"https://github.com/zhxieml/remiss-jailbreak/blob/HEAD/src/remissopt.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bf9d280eb54736b7"}},{"code_sha256_prefix":"e1f298fad7ae62ed","entry":"evaluate_next_token_candidates","repo":"zhxieml/remiss-jailbreak","repo_kind":"official","path":"src/remissopt.py","file_url":"https://github.com/zhxieml/remiss-jailbreak/blob/HEAD/src/remissopt.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e1f298fad7ae62ed"}},{"code_sha256_prefix":"02fbdc293c7c022f","entry":"get_next_token_probabilities","repo":"zhxieml/remiss-jailbreak","repo_kind":"official","path":"src/remissopt.py","file_url":"https://github.com/zhxieml/remiss-jailbreak/blob/HEAD/src/remissopt.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"02fbdc293c7c022f"}},{"code_sha256_prefix":"9f311245e1b7a581","entry":"reMissOpt","repo":"zhxieml/remiss-jailbreak","repo_kind":"official","path":"src/remissopt.py","file_url":"https://github.com/zhxieml/remiss-jailbreak/blob/HEAD/src/remissopt.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9f311245e1b7a581"}},{"code_sha256_prefix":"4d986a6e6f39f5ce","entry":"select_and_evaluate_next_token_candidates","repo":"zhxieml/remiss-jailbreak","repo_kind":"official","path":"src/remissopt.py","file_url":"https://github.com/zhxieml/remiss-jailbreak/blob/HEAD/src/remissopt.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4d986a6e6f39f5ce"}},{"code_sha256_prefix":"09209b225f2ff6e0","entry":"select_next_beams","repo":"zhxieml/remiss-jailbreak","repo_kind":"official","path":"src/remissopt.py","file_url":"https://github.com/zhxieml/remiss-jailbreak/blob/HEAD/src/remissopt.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"09209b225f2ff6e0"}},{"code_sha256_prefix":"773e7f99aa71d7e3","entry":"stack_seqs","repo":"zhxieml/remiss-jailbreak","repo_kind":"official","path":"src/remissopt.py","file_url":"https://github.com/zhxieml/remiss-jailbreak/blob/HEAD/src/remissopt.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"773e7f99aa71d7e3"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}