{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/transcoders-beat-sparse-autoencoders-for","title":"Transcoders Beat Sparse Autoencoders for Interpretability","arxiv_id":"2501.18823","date":"2025-01-31","proceeding":null,"authors":["Gonçalo Paulo","Stepan Shabalin","Nora Belrose"],"abstract":"Sparse autoencoders (SAEs) extract human-interpretable features from deep neural networks by transforming their activations into a sparse, higher dimensional latent space, and then reconstructing the activations from these latents. Transcoders are similar to SAEs, but they are trained to reconstruct the output of a component of a deep network given its input. In this work, we compare the features found by transcoders and SAEs trained on the same model and data, finding that transcoder features are significantly more interpretable. We also propose _skip transcoders_, which add an affine skip connection to the transcoder architecture, and show that these achieve lower reconstruction loss with no effect on interpretability.","url_abs":"https://arxiv.org/abs/2501.18823v1","url_pdf":"https://arxiv.org/pdf/2501.18823v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"transcoders-beat-sparse-autoencoders-for","repo_url":"https://github.com/eleutherai/sae","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2501.18823","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2501.18823"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/eleutherai/sae","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":8},"by_repo_kind":{"official":{"samples":8,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"94434469ea2ebc0d","entry":"assert_type","repo":"eleutherai/sae","repo_kind":"official","path":"sparsify/utils.py","file_url":"https://github.com/eleutherai/sae/blob/HEAD/sparsify/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"94434469ea2ebc0d"}},{"code_sha256_prefix":"3fafa18bde72510f","entry":"chunk_and_tokenize","repo":"eleutherai/sae","repo_kind":"official","path":"sparsify/data.py","file_url":"https://github.com/eleutherai/sae/blob/HEAD/sparsify/data.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3fafa18bde72510f"}},{"code_sha256_prefix":"c839c072b645f3c8","entry":"fused_encoder","repo":"eleutherai/sae","repo_kind":"official","path":"sparsify/fused_encoder.py","file_url":"https://github.com/eleutherai/sae/blob/HEAD/sparsify/fused_encoder.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c839c072b645f3c8"}},{"code_sha256_prefix":"b44a86959a6f33aa","entry":"get_columns_all_equal","repo":"eleutherai/sae","repo_kind":"official","path":"sparsify/data.py","file_url":"https://github.com/eleutherai/sae/blob/HEAD/sparsify/data.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b44a86959a6f33aa"}},{"code_sha256_prefix":"72500992c94f5911","entry":"get_layer_list","repo":"eleutherai/sae","repo_kind":"official","path":"sparsify/utils.py","file_url":"https://github.com/eleutherai/sae/blob/HEAD/sparsify/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"72500992c94f5911"}},{"code_sha256_prefix":"55dfcfdde405bda2","entry":"get_saes_by_layer_name","repo":"eleutherai/sae","repo_kind":"official","path":"sparsify/trainer.py","file_url":"https://github.com/eleutherai/sae/blob/HEAD/sparsify/trainer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"55dfcfdde405bda2"}},{"code_sha256_prefix":"1930bbdc2f82dc62","entry":"quintic_newtonschulz","repo":"eleutherai/sae","repo_kind":"official","path":"sparsify/muon.py","file_url":"https://github.com/eleutherai/sae/blob/HEAD/sparsify/muon.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1930bbdc2f82dc62"}},{"code_sha256_prefix":"4cd2443146d52623","entry":"resolve_widths","repo":"eleutherai/sae","repo_kind":"official","path":"sparsify/utils.py","file_url":"https://github.com/eleutherai/sae/blob/HEAD/sparsify/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4cd2443146d52623"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}