{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/sparse-autoencoders-learn-monosemantic","title":"Sparse Autoencoders Learn Monosemantic Features in Vision-Language Models","arxiv_id":"2504.02821","date":"2025-04-03","proceeding":null,"authors":["Mateusz Pach","Shyamgopal Karthik","Quentin Bouniot","Serge Belongie","Zeynep Akata"],"abstract":"Sparse Autoencoders (SAEs) have recently been shown to enhance interpretability and steerability in Large Language Models (LLMs). In this work, we extend the application of SAEs to Vision-Language Models (VLMs), such as CLIP, and introduce a comprehensive framework for evaluating monosemanticity in vision representations. Our experimental results reveal that SAEs trained on VLMs significantly enhance the monosemanticity of individual neurons while also exhibiting hierarchical representations that align well with expert-defined structures (e.g., iNaturalist taxonomy). Most notably, we demonstrate that applying SAEs to intervene on a CLIP vision encoder, directly steer output from multimodal LLMs (e.g., LLaVA) without any modifications to the underlying model. These findings emphasize the practicality and efficacy of SAEs as an unsupervised approach for enhancing both the interpretability and control of VLMs.","url_abs":"https://arxiv.org/abs/2504.02821v1","url_pdf":"https://arxiv.org/pdf/2504.02821v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"sparse-autoencoders-learn-monosemantic","repo_url":"https://github.com/explainableml/sae-for-vlm","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[],"methods":[{"method_slug":"align","method_name":"ALIGN"},{"method_slug":"clip","method_name":"CLIP"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2504.02821","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2504.02821"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/ExplainableML/sae-for-vlm","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/explainableml/sae-for-vlm","reach":{"status":"ok"}}],"summary":{"ran":3,"ran_fixture":3,"ran_draft_wrong":2,"unverified":2},"by_repo_kind":{"official":{"samples":10,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":10,"samples":[{"code_sha256_prefix":"85c251f29a71252f","entry":"AutoEncoder","repo":"explainableml/sae-for-vlm","repo_kind":"official","path":"dictionary_learning/trainers/standard.py","file_url":"https://github.com/explainableml/sae-for-vlm/blob/HEAD/dictionary_learning/trainers/standard.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"85c251f29a71252f"}},{"code_sha256_prefix":"e73aefb114ff270d","entry":"ConstrainedAdam","repo":"explainableml/sae-for-vlm","repo_kind":"official","path":"dictionary_learning/trainers/standard.py","file_url":"https://github.com/explainableml/sae-for-vlm/blob/HEAD/dictionary_learning/trainers/standard.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e73aefb114ff270d"}},{"code_sha256_prefix":"2d45e67a3164c483","entry":"SAETrainer","repo":"explainableml/sae-for-vlm","repo_kind":"official","path":"dictionary_learning/trainers/standard.py","file_url":"https://github.com/explainableml/sae-for-vlm/blob/HEAD/dictionary_learning/trainers/standard.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2d45e67a3164c483"}},{"code_sha256_prefix":"5207afb32bd9e1ae","entry":"get_lr_schedule","repo":"explainableml/sae-for-vlm","repo_kind":"official","path":"dictionary_learning/trainers/trainer.py","file_url":"https://github.com/explainableml/sae-for-vlm/blob/HEAD/dictionary_learning/trainers/trainer.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5207afb32bd9e1ae"}},{"code_sha256_prefix":"23d3c9ea7f5c1860","entry":"get_lr_schedule","repo":"explainableml/sae-for-vlm","repo_kind":"official","path":"dictionary_learning/trainers/standard.py","file_url":"https://github.com/explainableml/sae-for-vlm/blob/HEAD/dictionary_learning/trainers/standard.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"23d3c9ea7f5c1860"}},{"code_sha256_prefix":"991634aab38ccd6b","entry":"get_sparsity_warmup_fn","repo":"explainableml/sae-for-vlm","repo_kind":"official","path":"dictionary_learning/trainers/standard.py","file_url":"https://github.com/explainableml/sae-for-vlm/blob/HEAD/dictionary_learning/trainers/standard.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"991634aab38ccd6b"}},{"code_sha256_prefix":"7c278b430560062c","entry":"remove_gradient_parallel_to_decoder_directions","repo":"explainableml/sae-for-vlm","repo_kind":"official","path":"dictionary_learning/trainers/trainer.py","file_url":"https://github.com/explainableml/sae-for-vlm/blob/HEAD/dictionary_learning/trainers/trainer.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7c278b430560062c"}},{"code_sha256_prefix":"4809ea21f71004dd","entry":"set_decoder_norm_to_unit_norm","repo":"explainableml/sae-for-vlm","repo_kind":"official","path":"dictionary_learning/trainers/trainer.py","file_url":"https://github.com/explainableml/sae-for-vlm/blob/HEAD/dictionary_learning/trainers/trainer.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4809ea21f71004dd"}},{"code_sha256_prefix":"97d1ef510b2903c9","entry":"Dictionary","repo":"explainableml/sae-for-vlm","repo_kind":"official","path":"dictionary_learning/trainers/standard.py","file_url":"https://github.com/explainableml/sae-for-vlm/blob/HEAD/dictionary_learning/trainers/standard.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"97d1ef510b2903c9"}},{"code_sha256_prefix":"67ef7307750b0078","entry":"StandardTrainer","repo":"explainableml/sae-for-vlm","repo_kind":"official","path":"dictionary_learning/trainers/standard.py","file_url":"https://github.com/explainableml/sae-for-vlm/blob/HEAD/dictionary_learning/trainers/standard.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"67ef7307750b0078"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}