{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/depn-detecting-and-editing-privacy-neurons-in","title":"DEPN: Detecting and Editing Privacy Neurons in Pretrained Language Models","arxiv_id":"2310.20138","date":"2023-10-31","proceeding":null,"authors":["Xinwei Wu","Junzhuo Li","Minghui Xu","Weilong Dong","Shuangzhi Wu","Chao Bian","Deyi Xiong"],"abstract":"Large language models pretrained on a huge amount of data capture rich knowledge and information in the training data. The ability of data memorization and regurgitation in pretrained language models, revealed in previous studies, brings the risk of data leakage. In order to effectively reduce these risks, we propose a framework DEPN to Detect and Edit Privacy Neurons in pretrained language models, partially inspired by knowledge neurons and model editing. In DEPN, we introduce a novel method, termed as privacy neuron detector, to locate neurons associated with private information, and then edit these detected privacy neurons by setting their activations to zero. Furthermore, we propose a privacy neuron aggregator dememorize private information in a batch processing manner. Experimental results show that our method can significantly and efficiently reduce the exposure of private data leakage without deteriorating the performance of the model. Additionally, we empirically demonstrate the relationship between model memorization and privacy neurons, from multiple perspectives, including model size, training time, prompts, privacy neuron distribution, illustrating the robustness of our approach.","url_abs":"https://arxiv.org/abs/2310.20138v2","url_pdf":"https://arxiv.org/pdf/2310.20138v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"depn-detecting-and-editing-privacy-neurons-in","repo_url":"https://github.com/flamewei123/DEPN","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"memorization","task_name":"Memorization"},{"task_slug":"model-editing","task_name":"Model Editing"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2310.20138","atlas_url":"https://app.syntology.ai/?focus=2310.20138","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.20138"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/flamewei123/DEPN","reach":{"status":"ok"}}],"summary":{"ran_draft_wrong":2,"ran_honours":3,"ran":3,"unverified":1},"by_repo_kind":{"official":{"samples":9,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":9,"samples":[{"code_sha256_prefix":"0f786c407fb1ee4c","entry":"swish","repo":"flamewei123/DEPN","repo_kind":"official","path":"src/custom_bert.py","file_url":"https://github.com/flamewei123/DEPN/blob/HEAD/src/custom_bert.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":2,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0f786c407fb1ee4c"}},{"code_sha256_prefix":"7ca5f6affb7142af","entry":"convert_to_triplet_ig","repo":"flamewei123/DEPN","repo_kind":"official","path":"src/1_calculate_attribution.py","file_url":"https://github.com/flamewei123/DEPN/blob/HEAD/src/1_calculate_attribution.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7ca5f6affb7142af"}},{"code_sha256_prefix":"359b3d77e480e6f8","entry":"eval_ppl","repo":"flamewei123/DEPN","repo_kind":"official","path":"src/3_edit_privacy_neurons.py","file_url":"https://github.com/flamewei123/DEPN/blob/HEAD/src/3_edit_privacy_neurons.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"359b3d77e480e6f8"}},{"code_sha256_prefix":"ec86d7c6151f7d68","entry":"example2feature","repo":"flamewei123/DEPN","repo_kind":"official","path":"src/1_calculate_attribution.py","file_url":"https://github.com/flamewei123/DEPN/blob/HEAD/src/1_calculate_attribution.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ec86d7c6151f7d68"}},{"code_sha256_prefix":"56a9ab06b860b180","entry":"gelu","repo":"flamewei123/DEPN","repo_kind":"official","path":"src/custom_bert.py","file_url":"https://github.com/flamewei123/DEPN/blob/HEAD/src/custom_bert.py","link_basis":"plan_row","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"56a9ab06b860b180"}},{"code_sha256_prefix":"dbb7a4fa7b6bb8d1","entry":"pos_list2str","repo":"flamewei123/DEPN","repo_kind":"official","path":"src/2_filter_privacy_neurons.py","file_url":"https://github.com/flamewei123/DEPN/blob/HEAD/src/2_filter_privacy_neurons.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"dbb7a4fa7b6bb8d1"}},{"code_sha256_prefix":"dae759b7f6c3b29d","entry":"pos_str2list","repo":"flamewei123/DEPN","repo_kind":"official","path":"src/2_filter_privacy_neurons.py","file_url":"https://github.com/flamewei123/DEPN/blob/HEAD/src/2_filter_privacy_neurons.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"dae759b7f6c3b29d"}},{"code_sha256_prefix":"9c42e668d2a4a9d7","entry":"scaled_input","repo":"flamewei123/DEPN","repo_kind":"official","path":"src/1_calculate_attribution.py","file_url":"https://github.com/flamewei123/DEPN/blob/HEAD/src/1_calculate_attribution.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9c42e668d2a4a9d7"}},{"code_sha256_prefix":"ee8a0be5a85129de","entry":"load_evaldata","repo":"flamewei123/DEPN","repo_kind":"official","path":"src/3_edit_privacy_neurons.py","file_url":"https://github.com/flamewei123/DEPN/blob/HEAD/src/3_edit_privacy_neurons.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ee8a0be5a85129de"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}