{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/decomposing-and-interpreting-image","title":"Decomposing and Interpreting Image Representations via Text in ViTs Beyond CLIP","arxiv_id":"2406.01583","date":"2024-06-03","proceeding":null,"authors":["Sriram Balasubramanian","Samyadeep Basu","Soheil Feizi"],"abstract":"Recent work has explored how individual components of the CLIP-ViT model contribute to the final representation by leveraging the shared image-text representation space of CLIP. These components, such as attention heads and MLPs, have been shown to capture distinct image features like shape, color or texture. However, understanding the role of these components in arbitrary vision transformers (ViTs) is challenging. To this end, we introduce a general framework which can identify the roles of various components in ViTs beyond CLIP. Specifically, we (a) automate the decomposition of the final representation into contributions from different model components, and (b) linearly map these contributions to CLIP space to interpret them via text. Additionally, we introduce a novel scoring function to rank components by their importance with respect to specific features. Applying our framework to various ViT variants (e.g. DeiT, DINO, DINOv2, Swin, MaxViT), we gain insights into the roles of different components concerning particular image features. These insights facilitate applications such as image retrieval using text descriptions or reference images, visualizing token importance heatmaps, and mitigating spurious correlations. We release our code to reproduce the experiments at https://github.com/SriramB-98/vit-decompose","url_abs":"https://arxiv.org/abs/2406.01583v2","url_pdf":"https://arxiv.org/pdf/2406.01583v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"decomposing-and-interpreting-image","repo_url":"https://github.com/sriramb-98/vit-decompose","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"image-retrieval","task_name":"Image Retrieval"}],"methods":[{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"clip","method_name":"CLIP"},{"method_slug":"dino","method_name":"DINO"},{"method_slug":"deit","method_name":"DeiT"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"feedforward-network","method_name":"Feedforward Network"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"vision-transformer","method_name":"Vision Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2406.01583","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.01583"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/SriramB-98/vit-decompose","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sriramb-98/vit-decompose","reach":{"status":"ok"}}],"summary":{"ran":9,"ran_honours":1,"unverified":3},"by_repo_kind":{"official":{"samples":13,"ran":10,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":13,"samples":[{"code_sha256_prefix":"ad1f8ca3394f8a3e","entry":"collect_layer_components","repo":"SriramB-98/vit-decompose","repo_kind":"official","path":"helpers/model_utils.py","file_url":"https://github.com/SriramB-98/vit-decompose/blob/HEAD/helpers/model_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ad1f8ca3394f8a3e"}},{"code_sha256_prefix":"af8eeaf806f4ef88","entry":"concat_dict","repo":"SriramB-98/vit-decompose","repo_kind":"official","path":"helpers/decompose_utils.py","file_url":"https://github.com/SriramB-98/vit-decompose/blob/HEAD/helpers/decompose_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"af8eeaf806f4ef88"}},{"code_sha256_prefix":"bb93aacaffbcc8d1","entry":"detach_input","repo":"SriramB-98/vit-decompose","repo_kind":"official","path":"helpers/model_utils.py","file_url":"https://github.com/SriramB-98/vit-decompose/blob/HEAD/helpers/model_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bb93aacaffbcc8d1"}},{"code_sha256_prefix":"fac353c620fc2b66","entry":"dict_to_tensor","repo":"SriramB-98/vit-decompose","repo_kind":"official","path":"helpers/decompose_utils.py","file_url":"https://github.com/SriramB-98/vit-decompose/blob/HEAD/helpers/decompose_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fac353c620fc2b66"}},{"code_sha256_prefix":"1984b228b8b7a70e","entry":"extract_shape_metadata","repo":"SriramB-98/vit-decompose","repo_kind":"official","path":"helpers/linear_decompose.py","file_url":"https://github.com/SriramB-98/vit-decompose/blob/HEAD/helpers/linear_decompose.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1984b228b8b7a70e"}},{"code_sha256_prefix":"dd461c55cc3b4e57","entry":"get_exec_order","repo":"sriramb-98/vit-decompose","repo_kind":"official","path":"component_ablation.py","file_url":"https://github.com/sriramb-98/vit-decompose/blob/HEAD/component_ablation.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"dd461c55cc3b4e57"}},{"code_sha256_prefix":"696c02b1ccc49929","entry":"head_outputs","repo":"SriramB-98/vit-decompose","repo_kind":"official","path":"helpers/decompose_utils.py","file_url":"https://github.com/SriramB-98/vit-decompose/blob/HEAD/helpers/decompose_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"696c02b1ccc49929"}},{"code_sha256_prefix":"daf19f974b1c8d50","entry":"list_to_tensor","repo":"SriramB-98/vit-decompose","repo_kind":"official","path":"helpers/model_utils.py","file_url":"https://github.com/SriramB-98/vit-decompose/blob/HEAD/helpers/model_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"daf19f974b1c8d50"}},{"code_sha256_prefix":"f008b9c99180b5f5","entry":"prod","repo":"SriramB-98/vit-decompose","repo_kind":"official","path":"helpers/linear_decompose.py","file_url":"https://github.com/SriramB-98/vit-decompose/blob/HEAD/helpers/linear_decompose.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f008b9c99180b5f5"}},{"code_sha256_prefix":"5f3750bcdd0d49e0","entry":"split","repo":"SriramB-98/vit-decompose","repo_kind":"official","path":"helpers/linear_decompose.py","file_url":"https://github.com/SriramB-98/vit-decompose/blob/HEAD/helpers/linear_decompose.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5f3750bcdd0d49e0"}},{"code_sha256_prefix":"3f5d672bc6ebb458","entry":"checkpoint_filter_fn","repo":"SriramB-98/vit-decompose","repo_kind":"official","path":"timm/models/tiny_vit.py","file_url":"https://github.com/SriramB-98/vit-decompose/blob/HEAD/timm/models/tiny_vit.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3f5d672bc6ebb458"}},{"code_sha256_prefix":"310ebed22b298ee3","entry":"get_imgs_across_heads","repo":"sriramb-98/vit-decompose","repo_kind":"official","path":"image_retrieval_from_image.py","file_url":"https://github.com/sriramb-98/vit-decompose/blob/HEAD/image_retrieval_from_image.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"310ebed22b298ee3"}},{"code_sha256_prefix":"6236e4d25c0310db","entry":"get_imgs_across_heads","repo":"sriramb-98/vit-decompose","repo_kind":"official","path":"image_retrieval_from_text.py","file_url":"https://github.com/sriramb-98/vit-decompose/blob/HEAD/image_retrieval_from_text.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6236e4d25c0310db"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}