{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/proxyclip-proxy-attention-improves-clip-for","title":"ProxyCLIP: Proxy Attention Improves CLIP for Open-Vocabulary Segmentation","arxiv_id":"2408.04883","date":"2024-08-09","proceeding":null,"authors":["Mengcheng Lan","Chaofeng Chen","Yiping Ke","Xinjiang Wang","Litong Feng","Wayne Zhang"],"abstract":"Open-vocabulary semantic segmentation requires models to effectively integrate visual representations with open-vocabulary semantic labels. While Contrastive Language-Image Pre-training (CLIP) models shine in recognizing visual concepts from text, they often struggle with segment coherence due to their limited localization ability. In contrast, Vision Foundation Models (VFMs) excel at acquiring spatially consistent local visual representations, yet they fall short in semantic understanding. This paper introduces ProxyCLIP, an innovative framework designed to harmonize the strengths of both CLIP and VFMs, facilitating enhanced open-vocabulary semantic segmentation. ProxyCLIP leverages the spatial feature correspondence from VFMs as a form of proxy attention to augment CLIP, thereby inheriting the VFMs' robust local consistency and maintaining CLIP's exceptional zero-shot transfer capacity. We propose an adaptive normalization and masking strategy to get the proxy attention from VFMs, allowing for adaptation across different VFMs. Remarkably, as a training-free approach, ProxyCLIP significantly improves the average mean Intersection over Union (mIoU) across eight benchmarks from 40.3 to 44.4, showcasing its exceptional efficacy in bridging the gap between spatial precision and semantic richness for the open-vocabulary segmentation task.","url_abs":"https://arxiv.org/abs/2408.04883v1","url_pdf":"https://arxiv.org/pdf/2408.04883v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"proxyclip-proxy-attention-improves-clip-for","repo_url":"https://github.com/mc-lan/proxyclip","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"open-vocabulary-semantic-segmentation","task_name":"Open Vocabulary Semantic Segmentation"},{"task_slug":"open-vocabulary-semantic-segmentation-1","task_name":"Open-Vocabulary Semantic Segmentation"},{"task_slug":"segmentation","task_name":"Segmentation"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"},{"task_slug":"unsupervised-semantic-segmentation-with","task_name":"Unsupervised Semantic Segmentation with Language-image Pre-training"}],"methods":[{"method_slug":"attention","method_name":"Attention"},{"method_slug":"clip","method_name":"CLIP"},{"method_slug":"softmax","method_name":"Softmax"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/unsupervised-semantic-segmentation-with-4","task":"Unsupervised Semantic Segmentation with Language-image Pre-training","dataset":"ADE20K","model":"ProxyCLIP","rank_in_archive_order":4,"of":13,"metrics":{"Mean IoU (val)":"24.2"},"uses_additional_data":false},{"leaderboard":"/sota/unsupervised-semantic-segmentation-with-10","task":"Unsupervised Semantic Segmentation with Language-image Pre-training","dataset":"COCO-Object","model":"ProxyCLIP","rank_in_archive_order":3,"of":12,"metrics":{"mIoU":"39.2"},"uses_additional_data":false},{"leaderboard":"/sota/unsupervised-semantic-segmentation-with-9","task":"Unsupervised Semantic Segmentation with Language-image Pre-training","dataset":"COCO-Stuff-171","model":"ProxyCLIP","rank_in_archive_order":4,"of":12,"metrics":{"mIoU":"26.8"},"uses_additional_data":false},{"leaderboard":"/sota/unsupervised-semantic-segmentation-with-3","task":"Unsupervised Semantic Segmentation with Language-image Pre-training","dataset":"Cityscapes val","model":"ProxyCLIP","rank_in_archive_order":3,"of":12,"metrics":{"mIoU":"42.0"},"uses_additional_data":false},{"leaderboard":"/sota/unsupervised-semantic-segmentation-with-8","task":"Unsupervised Semantic Segmentation with Language-image Pre-training","dataset":"PASCAL Context-59","model":"ProxyCLIP","rank_in_archive_order":4,"of":12,"metrics":{"mIoU":"39.6"},"uses_additional_data":false},{"leaderboard":"/sota/unsupervised-semantic-segmentation-with-12","task":"Unsupervised Semantic Segmentation with Language-image Pre-training","dataset":"PASCAL Context-60","model":"ProxyCLIP","rank_in_archive_order":4,"of":4,"metrics":{"mIoU":"35.4"},"uses_additional_data":false},{"leaderboard":"/sota/unsupervised-semantic-segmentation-with-11","task":"Unsupervised Semantic Segmentation with Language-image Pre-training","dataset":"PASCAL VOC","model":"ProxyCLIP","rank_in_archive_order":5,"of":10,"metrics":{"mIoU":"65.0"},"uses_additional_data":false},{"leaderboard":"/sota/unsupervised-semantic-segmentation-with-7","task":"Unsupervised Semantic Segmentation with Language-image Pre-training","dataset":"PascalVOC-20","model":"ProxyCLIP","rank_in_archive_order":5,"of":10,"metrics":{"mIoU":"83.3"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2408.04883","atlas_url":"https://app.syntology.ai/?focus=2408.04883","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2408.04883"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mc-lan/proxyclip","reach":{"status":"ok"}}],"summary":{"ran_violates":1,"ran_draft_wrong":2,"unverified":6},"by_repo_kind":{"official":{"samples":9,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":9,"samples":[{"code_sha256_prefix":"f19962ebb134b3d7","entry":"convert_to_custom_text_state_dict","repo":"mc-lan/proxyclip","repo_kind":"official","path":"open_clip/model.py","file_url":"https://github.com/mc-lan/proxyclip/blob/HEAD/open_clip/model.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f19962ebb134b3d7"}},{"code_sha256_prefix":"dcd422d66b0581d8","entry":"get_cast_dtype","repo":"mc-lan/proxyclip","repo_kind":"official","path":"open_clip/model.py","file_url":"https://github.com/mc-lan/proxyclip/blob/HEAD/open_clip/model.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"dcd422d66b0581d8"}},{"code_sha256_prefix":"b476c8cfbf0f1f47","entry":"get_input_dtype","repo":"mc-lan/proxyclip","repo_kind":"official","path":"open_clip/model.py","file_url":"https://github.com/mc-lan/proxyclip/blob/HEAD/open_clip/model.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b476c8cfbf0f1f47"}},{"code_sha256_prefix":"ddcbd45e940484ee","entry":"gather_features","repo":"mc-lan/proxyclip","repo_kind":"official","path":"open_clip/loss.py","file_url":"https://github.com/mc-lan/proxyclip/blob/HEAD/open_clip/loss.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ddcbd45e940484ee"}},{"code_sha256_prefix":"54731d24b3557840","entry":"get_cls_idx","repo":"mc-lan/proxyclip","repo_kind":"official","path":"proxyclip_segmentor.py","file_url":"https://github.com/mc-lan/proxyclip/blob/HEAD/proxyclip_segmentor.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"54731d24b3557840"}},{"code_sha256_prefix":"e332856e3c2fc814","entry":"neighbour_exchange","repo":"mc-lan/proxyclip","repo_kind":"official","path":"open_clip/loss.py","file_url":"https://github.com/mc-lan/proxyclip/blob/HEAD/open_clip/loss.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e332856e3c2fc814"}},{"code_sha256_prefix":"5b1fd364afcf3c05","entry":"neighbour_exchange_bidir","repo":"mc-lan/proxyclip","repo_kind":"official","path":"open_clip/loss.py","file_url":"https://github.com/mc-lan/proxyclip/blob/HEAD/open_clip/loss.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5b1fd364afcf3c05"}},{"code_sha256_prefix":"fb651d0a97fd4d3f","entry":"prepare_inputs_for_generation","repo":"mc-lan/proxyclip","repo_kind":"official","path":"open_clip/coca_model.py","file_url":"https://github.com/mc-lan/proxyclip/blob/HEAD/open_clip/coca_model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fb651d0a97fd4d3f"}},{"code_sha256_prefix":"2a377da4a76a2d44","entry":"register_pooler","repo":"mc-lan/proxyclip","repo_kind":"official","path":"open_clip/hf_model.py","file_url":"https://github.com/mc-lan/proxyclip/blob/HEAD/open_clip/hf_model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2a377da4a76a2d44"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}