{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/ablation-is-not-enough-to-emulate-dpo-how","title":"Beyond Toxic Neurons: A Mechanistic Analysis of DPO for Toxicity Reduction","arxiv_id":"2411.06424","date":"2024-11-10","proceeding":null,"authors":["Yushi Yang","Filip Sondej","Harry Mayne","Adam Mahdi"],"abstract":"Safety fine-tuning algorithms are widely used to reduce harmful outputs in language models, but how they achieve this remain unclear. Studying the Direct Preference Optimization (DPO) algorithm for toxicity reduction, current explanations claim that DPO achieves this by dampening the activations of toxic MLP neurons. However, through activation patching, we show that this explanation is incomplete. Projections onto a toxicity probe's direction show that only 4.9% of toxicity reduction comes from dampened toxic neurons. Instead, DPO reduces toxicity through distributed activation shifts across a majority of neurons, progressively shifting MLP layer outputs away from toxicity. These shifts accumulate across four neuron groups: two reducing toxicity and two promoting anti-toxicity. Activation patching validates the cumulative roles of these groups, where patching all identified groups effectively replicates DPO's effects. These findings illustrate DPO's mechanism: it reduces toxicity by accumulating small activation shifts across many neurons throughout the layers. Our findings provide new mechanistic insights into how safety fine-tuning reduces harmful outputs in language models.","url_abs":"https://arxiv.org/abs/2411.06424v2","url_pdf":"https://arxiv.org/pdf/2411.06424v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"ablation-is-not-enough-to-emulate-dpo-how","repo_url":"https://github.com/yushi-y/dpo-toxic-neurons","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[],"methods":[{"method_slug":"dpo","method_name":"DPO"},{"method_slug":"patching","method_name":"Patching"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2411.06424","atlas_url":"https://app.syntology.ai/?focus=2411.06424","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2411.06424"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yushi-y/dpo-toxic-neurons","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":15},"by_repo_kind":{"official":{"samples":15,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"6e420696373414b1","entry":"dpo_loss","repo":"yushi-y/dpo-toxic-neurons","repo_kind":"official","path":"train_dpo/trainers.py","file_url":"https://github.com/yushi-y/dpo-toxic-neurons/blob/HEAD/train_dpo/trainers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6e420696373414b1"}},{"code_sha256_prefix":"813d811b5f586d21","entry":"get_gold","repo":"yushi-y/dpo-toxic-neurons","repo_kind":"official","path":"evaluation/generate_funcs.py","file_url":"https://github.com/yushi-y/dpo-toxic-neurons/blob/HEAD/evaluation/generate_funcs.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"813d811b5f586d21"}},{"code_sha256_prefix":"9df69409dc0ffe78","entry":"get_intervene_vector","repo":"yushi-y/dpo-toxic-neurons","repo_kind":"official","path":"activation_patching/hook_utils.py","file_url":"https://github.com/yushi-y/dpo-toxic-neurons/blob/HEAD/activation_patching/hook_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9df69409dc0ffe78"}},{"code_sha256_prefix":"573680795434164c","entry":"get_kl_div","repo":"yushi-y/dpo-toxic-neurons","repo_kind":"official","path":"train_dpo/trainers.py","file_url":"https://github.com/yushi-y/dpo-toxic-neurons/blob/HEAD/train_dpo/trainers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"573680795434164c"}},{"code_sha256_prefix":"8ab43db002bf984d","entry":"get_local_dir","repo":"yushi-y/dpo-toxic-neurons","repo_kind":"official","path":"train_dpo/dpo_utils.py","file_url":"https://github.com/yushi-y/dpo-toxic-neurons/blob/HEAD/train_dpo/dpo_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8ab43db002bf984d"}},{"code_sha256_prefix":"1df5ede8f8950a70","entry":"get_local_run_dir","repo":"yushi-y/dpo-toxic-neurons","repo_kind":"official","path":"train_dpo/dpo_utils.py","file_url":"https://github.com/yushi-y/dpo-toxic-neurons/blob/HEAD/train_dpo/dpo_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1df5ede8f8950a70"}},{"code_sha256_prefix":"d635938df605fbd3","entry":"get_prompts","repo":"yushi-y/dpo-toxic-neurons","repo_kind":"official","path":"evaluation/generate_funcs.py","file_url":"https://github.com/yushi-y/dpo-toxic-neurons/blob/HEAD/evaluation/generate_funcs.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d635938df605fbd3"}},{"code_sha256_prefix":"e7ce4c19888c6df6","entry":"get_remote_file","repo":"yushi-y/dpo-toxic-neurons","repo_kind":"official","path":"train_dpo/dpo_utils.py","file_url":"https://github.com/yushi-y/dpo-toxic-neurons/blob/HEAD/train_dpo/dpo_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e7ce4c19888c6df6"}},{"code_sha256_prefix":"527be56c58313107","entry":"get_svd_u_vec","repo":"yushi-y/dpo-toxic-neurons","repo_kind":"official","path":"activation_patching/hook_utils.py","file_url":"https://github.com/yushi-y/dpo-toxic-neurons/blob/HEAD/activation_patching/hook_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"527be56c58313107"}},{"code_sha256_prefix":"b4e9abfc1b4387cd","entry":"hook_subtract","repo":"yushi-y/dpo-toxic-neurons","repo_kind":"official","path":"activation_patching/hook_utils.py","file_url":"https://github.com/yushi-y/dpo-toxic-neurons/blob/HEAD/activation_patching/hook_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b4e9abfc1b4387cd"}},{"code_sha256_prefix":"cba91edd374a3e28","entry":"load_data","repo":"yushi-y/dpo-toxic-neurons","repo_kind":"official","path":"evaluation/eval_utils.py","file_url":"https://github.com/yushi-y/dpo-toxic-neurons/blob/HEAD/evaluation/eval_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cba91edd374a3e28"}},{"code_sha256_prefix":"89200b27db47f6ea","entry":"load_model","repo":"yushi-y/dpo-toxic-neurons","repo_kind":"official","path":"evaluation/eval_utils.py","file_url":"https://github.com/yushi-y/dpo-toxic-neurons/blob/HEAD/evaluation/eval_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"89200b27db47f6ea"}},{"code_sha256_prefix":"f5813daae4f9b8f5","entry":"run_dummy","repo":"yushi-y/dpo-toxic-neurons","repo_kind":"official","path":"evaluation/metric_funcs.py","file_url":"https://github.com/yushi-y/dpo-toxic-neurons/blob/HEAD/evaluation/metric_funcs.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f5813daae4f9b8f5"}},{"code_sha256_prefix":"044b24804e21d67d","entry":"run_f1","repo":"yushi-y/dpo-toxic-neurons","repo_kind":"official","path":"evaluation/metric_funcs.py","file_url":"https://github.com/yushi-y/dpo-toxic-neurons/blob/HEAD/evaluation/metric_funcs.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"044b24804e21d67d"}},{"code_sha256_prefix":"1a98f49d7c56acc9","entry":"tokenize","repo":"yushi-y/dpo-toxic-neurons","repo_kind":"official","path":"evaluation/eval_utils.py","file_url":"https://github.com/yushi-y/dpo-toxic-neurons/blob/HEAD/evaluation/eval_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1a98f49d7c56acc9"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}