{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/applying-sparse-autoencoders-to-unlearn","title":"Applying sparse autoencoders to unlearn knowledge in language models","arxiv_id":"2410.19278","date":"2024-10-25","proceeding":null,"authors":["Eoin Farrell","Yeu-Tong Lau","Arthur Conmy"],"abstract":"We investigate whether sparse autoencoders (SAEs) can be used to remove knowledge from language models. We use the biology subset of the Weapons of Mass Destruction Proxy dataset and test on the gemma-2b-it and gemma-2-2b-it language models. We demonstrate that individual interpretable biology-related SAE features can be used to unlearn a subset of WMDP-Bio questions with minimal side-effects in domains other than biology. Our results suggest that negative scaling of feature activations is necessary and that zero ablating features is ineffective. We find that intervening using multiple SAE features simultaneously can unlearn multiple different topics, but with similar or larger unwanted side-effects than the existing Representation Misdirection for Unlearning technique. Current SAE quality or intervention techniques would need to improve to make SAE-based unlearning comparable to the existing fine-tuning based techniques.","url_abs":"https://arxiv.org/abs/2410.19278v2","url_pdf":"https://arxiv.org/pdf/2410.19278v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"applying-sparse-autoencoders-to-unlearn","repo_url":"https://github.com/efarrell1/train_sparse_autoencoder","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2410.19278","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.19278"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/efarrell1/train_sparse_autoencoder","reach":{"status":"ok"}}],"summary":{"unverified":10},"by_repo_kind":{"official":{"samples":10,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":10,"samples":[{"code_sha256_prefix":"bda89456f67fab15","entry":"compute_metrics","repo":"efarrell1/train_sparse_autoencoder","repo_kind":"official","path":"sae/utils.py","file_url":"https://github.com/efarrell1/train_sparse_autoencoder/blob/HEAD/sae/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bda89456f67fab15"}},{"code_sha256_prefix":"835a6e2fafdf1f20","entry":"compute_metrics_post","repo":"efarrell1/train_sparse_autoencoder","repo_kind":"official","path":"sae/metrics.py","file_url":"https://github.com/efarrell1/train_sparse_autoencoder/blob/HEAD/sae/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"835a6e2fafdf1f20"}},{"code_sha256_prefix":"37842cf203f39ef2","entry":"compute_simple_metrics","repo":"efarrell1/train_sparse_autoencoder","repo_kind":"official","path":"sae/metrics.py","file_url":"https://github.com/efarrell1/train_sparse_autoencoder/blob/HEAD/sae/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"37842cf203f39ef2"}},{"code_sha256_prefix":"71b34d41c30bc1db","entry":"create_config_inputs","repo":"efarrell1/train_sparse_autoencoder","repo_kind":"official","path":"sae/utils.py","file_url":"https://github.com/efarrell1/train_sparse_autoencoder/blob/HEAD/sae/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"71b34d41c30bc1db"}},{"code_sha256_prefix":"76f362a37c9f824c","entry":"generate_cfg_list","repo":"efarrell1/train_sparse_autoencoder","repo_kind":"official","path":"sae/config.py","file_url":"https://github.com/efarrell1/train_sparse_autoencoder/blob/HEAD/sae/config.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"76f362a37c9f824c"}},{"code_sha256_prefix":"9fe3a8d45e7196eb","entry":"list_of_dicts_to_dict_of_lists","repo":"efarrell1/train_sparse_autoencoder","repo_kind":"official","path":"sae/metrics.py","file_url":"https://github.com/efarrell1/train_sparse_autoencoder/blob/HEAD/sae/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9fe3a8d45e7196eb"}},{"code_sha256_prefix":"5b4fe03466e1e2e6","entry":"make_token_df","repo":"efarrell1/train_sparse_autoencoder","repo_kind":"official","path":"sae/analysis.py","file_url":"https://github.com/efarrell1/train_sparse_autoencoder/blob/HEAD/sae/analysis.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5b4fe03466e1e2e6"}},{"code_sha256_prefix":"72618a3a4fafef90","entry":"process_token","repo":"efarrell1/train_sparse_autoencoder","repo_kind":"official","path":"sae/analysis.py","file_url":"https://github.com/efarrell1/train_sparse_autoencoder/blob/HEAD/sae/analysis.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"72618a3a4fafef90"}},{"code_sha256_prefix":"b88bbf83718ecfec","entry":"process_tokens","repo":"efarrell1/train_sparse_autoencoder","repo_kind":"official","path":"sae/analysis.py","file_url":"https://github.com/efarrell1/train_sparse_autoencoder/blob/HEAD/sae/analysis.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b88bbf83718ecfec"}},{"code_sha256_prefix":"de4343263a7e3497","entry":"round_1000","repo":"efarrell1/train_sparse_autoencoder","repo_kind":"official","path":"sae/utils.py","file_url":"https://github.com/efarrell1/train_sparse_autoencoder/blob/HEAD/sae/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"de4343263a7e3497"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}