{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/keeping-llms-aligned-after-fine-tuning-the","title":"Keeping LLMs Aligned After Fine-tuning: The Crucial Role of Prompt Templates","arxiv_id":"2402.18540","date":"2024-02-28","proceeding":null,"authors":["Kaifeng Lyu","Haoyu Zhao","Xinran Gu","Dingli Yu","Anirudh Goyal","Sanjeev Arora"],"abstract":"Public LLMs such as the Llama 2-Chat underwent alignment training and were considered safe. Recently Qi et al. [2024] reported that even benign fine-tuning on seemingly safe datasets can give rise to unsafe behaviors in the models. The current paper is about methods and best practices to mitigate such loss of alignment. We focus on the setting where a public model is fine-tuned before serving users for specific usage, where the model should improve on the downstream task while maintaining alignment. Through extensive experiments on several chat models (Meta's Llama 2-Chat, Mistral AI's Mistral 7B Instruct v0.2, and OpenAI's GPT-3.5 Turbo), this paper uncovers that the prompt templates used during fine-tuning and inference play a crucial role in preserving safety alignment, and proposes the ``Pure Tuning, Safe Testing'' (PTST) strategy -- fine-tune models without a safety prompt, but include it at test time. This seemingly counterintuitive strategy incorporates an intended distribution shift to encourage alignment preservation. Fine-tuning experiments on GSM8K, ChatDoctor, and OpenOrca show that PTST significantly reduces the rise of unsafe behaviors.","url_abs":"https://arxiv.org/abs/2402.18540v2","url_pdf":"https://arxiv.org/pdf/2402.18540v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"keeping-llms-aligned-after-fine-tuning-the","repo_url":"https://github.com/vfleaking/ptst","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"gsm8k","task_name":"GSM8K"},{"task_slug":"safety-alignment","task_name":"Safety Alignment"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"cosine-annealing","method_name":"Cosine Annealing"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"focus","method_name":"Focus"},{"method_slug":"gpt-3","method_name":"GPT-3"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-cosine-annealing","method_name":"Linear Warmup With Cosine Annealing"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2402.18540","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.18540"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/vfleaking/PTST","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/vfleaking/ptst","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":5,"ran_draft_wrong":1,"unverified":4},"by_repo_kind":{"official":{"samples":10,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"9dbc3a2ad725ec0b","entry":"get_prompt_template","repo":"vfleaking/PTST","repo_kind":"official","path":"gpt-api/prompt_utils.py","file_url":"https://github.com/vfleaking/PTST/blob/HEAD/gpt-api/prompt_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9dbc3a2ad725ec0b"}},{"code_sha256_prefix":"e7e4f43064424eca","entry":"question_read_csv","repo":"vfleaking/PTST","repo_kind":"official","path":"inference.py","file_url":"https://github.com/vfleaking/PTST/blob/HEAD/inference.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e7e4f43064424eca"}},{"code_sha256_prefix":"a8c390bfe1874cfc","entry":"question_read_json","repo":"vfleaking/PTST","repo_kind":"official","path":"inference.py","file_url":"https://github.com/vfleaking/PTST/blob/HEAD/inference.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a8c390bfe1874cfc"}},{"code_sha256_prefix":"f53a44fbf1d7997e","entry":"question_read_txt","repo":"vfleaking/PTST","repo_kind":"official","path":"inference.py","file_url":"https://github.com/vfleaking/PTST/blob/HEAD/inference.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f53a44fbf1d7997e"}},{"code_sha256_prefix":"64c1b0186895ea4a","entry":"read_jsonl","repo":"vfleaking/PTST","repo_kind":"official","path":"gpt-api/data_utils/prep_data.py","file_url":"https://github.com/vfleaking/PTST/blob/HEAD/gpt-api/data_utils/prep_data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"64c1b0186895ea4a"}},{"code_sha256_prefix":"8db5d6fa2677c64f","entry":"read_txt","repo":"vfleaking/ptst","repo_kind":"official","path":"llama2/lm_utils/prompt_utils.py","file_url":"https://github.com/vfleaking/ptst/blob/HEAD/llama2/lm_utils/prompt_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8db5d6fa2677c64f"}},{"code_sha256_prefix":"4424bf09213d140b","entry":"apply_prompt_template","repo":"vfleaking/PTST","repo_kind":"official","path":"prompt_utils.py","file_url":"https://github.com/vfleaking/PTST/blob/HEAD/prompt_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4424bf09213d140b"}},{"code_sha256_prefix":"5e33dbb2cc0e16a5","entry":"get_prompt_template","repo":"vfleaking/ptst","repo_kind":"official","path":"llama2/lm_utils/prompt_utils.py","file_url":"https://github.com/vfleaking/ptst/blob/HEAD/llama2/lm_utils/prompt_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5e33dbb2cc0e16a5"}},{"code_sha256_prefix":"1dfd5a6dff6a375b","entry":"get_prompt_template","repo":"vfleaking/PTST","repo_kind":"official","path":"prompt_utils.py","file_url":"https://github.com/vfleaking/PTST/blob/HEAD/prompt_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1dfd5a6dff6a375b"}},{"code_sha256_prefix":"abd14de7398ccffe","entry":"load_sharded_model_single_gpu","repo":"vfleaking/PTST","repo_kind":"official","path":"llama2/model_checkpointing/checkpoint_handler.py","file_url":"https://github.com/vfleaking/PTST/blob/HEAD/llama2/model_checkpointing/checkpoint_handler.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"abd14de7398ccffe"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}