{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/a-closer-look-at-machine-unlearning-for-large","title":"A Closer Look at Machine Unlearning for Large Language Models","arxiv_id":"2410.08109","date":"2024-10-10","proceeding":null,"authors":["Xiaojian Yuan","Tianyu Pang","Chao Du","Kejiang Chen","Weiming Zhang","Min Lin"],"abstract":"Large language models (LLMs) may memorize sensitive or copyrighted content, raising privacy and legal concerns. Due to the high cost of retraining from scratch, researchers attempt to employ machine unlearning to remove specific content from LLMs while preserving the overall performance. In this paper, we discuss several issues in machine unlearning for LLMs and provide our insights on possible approaches. To address the issue of inadequate evaluation of model outputs after unlearning, we introduce three additional metrics to evaluate token diversity, sentence semantics, and factual correctness. We then categorize unlearning methods into untargeted and targeted, and discuss their issues respectively. Specifically, the behavior that untargeted unlearning attempts to approximate is unpredictable and may involve hallucinations, and existing regularization is insufficient for targeted unlearning. To alleviate these issues, we propose using the objective of maximizing entropy (ME) for untargeted unlearning and incorporate answer preservation (AP) loss as regularization for targeted unlearning. Experimental results across three scenarios, i.e., fictitious unlearning, continual unlearning, and real-world unlearning, demonstrate the effectiveness of our approaches. The code is available at https://github.com/sail-sg/closer-look-LLM-unlearning.","url_abs":"https://arxiv.org/abs/2410.08109v4","url_pdf":"https://arxiv.org/pdf/2410.08109v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"a-closer-look-at-machine-unlearning-for-large","repo_url":"https://github.com/sail-sg/closer-look-llm-unlearning","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"diversity","task_name":"Diversity"},{"task_slug":"machine-unlearning","task_name":"Machine Unlearning"},{"task_slug":"sentence","task_name":"Sentence"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2410.08109","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.08109"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/sail-sg/closer-look-LLM-unlearning","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sail-sg/closer-look-llm-unlearning","reach":{"status":"ok"}}],"summary":{"ran":7,"ran_draft_wrong":1,"ran_fixture":4,"unverified":5},"by_repo_kind":{"official":{"samples":17,"ran":12,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":17,"samples":[{"code_sha256_prefix":"cdd171f20cd6f53d","entry":"collate_fn","repo":"sail-sg/closer-look-LLM-unlearning","repo_kind":"official","path":"dataset/data_module.py","file_url":"https://github.com/sail-sg/closer-look-LLM-unlearning/blob/HEAD/dataset/data_module.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"cdd171f20cd6f53d"}},{"code_sha256_prefix":"3953f493940e88b8","entry":"compute_token_entropy","repo":"sail-sg/closer-look-LLM-unlearning","repo_kind":"official","path":"metrics/metrics.py","file_url":"https://github.com/sail-sg/closer-look-LLM-unlearning/blob/HEAD/metrics/metrics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3953f493940e88b8"}},{"code_sha256_prefix":"649fc48067a48529","entry":"find_all_linear_names","repo":"sail-sg/closer-look-LLM-unlearning","repo_kind":"official","path":"forget.py","file_url":"https://github.com/sail-sg/closer-look-LLM-unlearning/blob/HEAD/forget.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"649fc48067a48529"}},{"code_sha256_prefix":"c2d4937262b0ac28","entry":"ga_loss","repo":"sail-sg/closer-look-llm-unlearning","repo_kind":"official","path":"trainer/losses.py","file_url":"https://github.com/sail-sg/closer-look-llm-unlearning/blob/HEAD/trainer/losses.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c2d4937262b0ac28"}},{"code_sha256_prefix":"a2f425992cbade01","entry":"general_eval","repo":"sail-sg/closer-look-LLM-unlearning","repo_kind":"official","path":"real_world_eval.py","file_url":"https://github.com/sail-sg/closer-look-LLM-unlearning/blob/HEAD/real_world_eval.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a2f425992cbade01"}},{"code_sha256_prefix":"01f406043cd388be","entry":"get_me_loss","repo":"sail-sg/closer-look-llm-unlearning","repo_kind":"official","path":"trainer/losses.py","file_url":"https://github.com/sail-sg/closer-look-llm-unlearning/blob/HEAD/trainer/losses.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"01f406043cd388be"}},{"code_sha256_prefix":"6c542acea5aa211c","entry":"get_total_len","repo":"sail-sg/closer-look-LLM-unlearning","repo_kind":"official","path":"utils/utils.py","file_url":"https://github.com/sail-sg/closer-look-LLM-unlearning/blob/HEAD/utils/utils.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6c542acea5aa211c"}},{"code_sha256_prefix":"340f1a6783f01f95","entry":"me_loss","repo":"sail-sg/closer-look-llm-unlearning","repo_kind":"official","path":"trainer/losses.py","file_url":"https://github.com/sail-sg/closer-look-llm-unlearning/blob/HEAD/trainer/losses.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"340f1a6783f01f95"}},{"code_sha256_prefix":"8076aab5782c4d02","entry":"merge_dicts","repo":"sail-sg/closer-look-LLM-unlearning","repo_kind":"official","path":"utils/utils.py","file_url":"https://github.com/sail-sg/closer-look-LLM-unlearning/blob/HEAD/utils/utils.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8076aab5782c4d02"}},{"code_sha256_prefix":"a3b77597ef1aa5c9","entry":"npo_loss","repo":"sail-sg/closer-look-llm-unlearning","repo_kind":"official","path":"trainer/losses.py","file_url":"https://github.com/sail-sg/closer-look-llm-unlearning/blob/HEAD/trainer/losses.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a3b77597ef1aa5c9"}},{"code_sha256_prefix":"10c5cb88157f0830","entry":"summary_results","repo":"sail-sg/closer-look-LLM-unlearning","repo_kind":"official","path":"real_world_eval.py","file_url":"https://github.com/sail-sg/closer-look-LLM-unlearning/blob/HEAD/real_world_eval.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"10c5cb88157f0830"}},{"code_sha256_prefix":"9b8e27aa12ba922b","entry":"token_entropy","repo":"sail-sg/closer-look-LLM-unlearning","repo_kind":"official","path":"metrics/metrics.py","file_url":"https://github.com/sail-sg/closer-look-LLM-unlearning/blob/HEAD/metrics/metrics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9b8e27aa12ba922b"}},{"code_sha256_prefix":"7dec325d088c0b76","entry":"convert_raw_data_to_model_format","repo":"sail-sg/closer-look-LLM-unlearning","repo_kind":"official","path":"dataset/data_module.py","file_url":"https://github.com/sail-sg/closer-look-LLM-unlearning/blob/HEAD/dataset/data_module.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7dec325d088c0b76"}},{"code_sha256_prefix":"28aa527377c5b4e8","entry":"convert_raw_forget_data_to_model_format","repo":"sail-sg/closer-look-LLM-unlearning","repo_kind":"official","path":"dataset/data_module.py","file_url":"https://github.com/sail-sg/closer-look-LLM-unlearning/blob/HEAD/dataset/data_module.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"28aa527377c5b4e8"}},{"code_sha256_prefix":"3c6bd6e091c76ca3","entry":"ga_loss","repo":"sail-sg/closer-look-LLM-unlearning","repo_kind":"official","path":"trainer/losses.py","file_url":"https://github.com/sail-sg/closer-look-LLM-unlearning/blob/HEAD/trainer/losses.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3c6bd6e091c76ca3"}},{"code_sha256_prefix":"0605aa76e6497ba3","entry":"get_loss","repo":"sail-sg/closer-look-llm-unlearning","repo_kind":"official","path":"trainer/losses.py","file_url":"https://github.com/sail-sg/closer-look-llm-unlearning/blob/HEAD/trainer/losses.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0605aa76e6497ba3"}},{"code_sha256_prefix":"06448b0513272463","entry":"get_model_identifiers_from_yaml","repo":"sail-sg/closer-look-LLM-unlearning","repo_kind":"official","path":"utils/utils.py","file_url":"https://github.com/sail-sg/closer-look-LLM-unlearning/blob/HEAD/utils/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"06448b0513272463"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}