{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/clip-diy-clip-dense-inference-yields-open","title":"CLIP-DIY: CLIP Dense Inference Yields Open-Vocabulary Semantic Segmentation For-Free","arxiv_id":"2309.14289","date":"2023-09-25","proceeding":null,"authors":["Monika Wysoczańska","Michaël Ramamonjisoa","Tomasz Trzciński","Oriane Siméoni"],"abstract":"The emergence of CLIP has opened the way for open-world image perception. The zero-shot classification capabilities of the model are impressive but are harder to use for dense tasks such as image segmentation. Several methods have proposed different modifications and learning schemes to produce dense output. Instead, we propose in this work an open-vocabulary semantic segmentation method, dubbed CLIP-DIY, which does not require any additional training or annotations, but instead leverages existing unsupervised object localization approaches. In particular, CLIP-DIY is a multi-scale approach that directly exploits CLIP classification abilities on patches of different sizes and aggregates the decision in a single map. We further guide the segmentation using foreground/background scores obtained using unsupervised object localization methods. With our method, we obtain state-of-the-art zero-shot semantic segmentation results on PASCAL VOC and perform on par with the best methods on COCO. The code is available at http://github.com/wysoczanska/clip-diy","url_abs":"https://arxiv.org/abs/2309.14289v2","url_pdf":"https://arxiv.org/pdf/2309.14289v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"clip-diy-clip-dense-inference-yields-open","repo_url":"https://github.com/wysoczanska/clip-diy","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"image-segmentation","task_name":"Image Segmentation"},{"task_slug":"object-localization","task_name":"Object Localization"},{"task_slug":"open-vocabulary-semantic-segmentation","task_name":"Open Vocabulary Semantic Segmentation"},{"task_slug":"open-vocabulary-semantic-segmentation-1","task_name":"Open-Vocabulary Semantic Segmentation"},{"task_slug":"segmentation","task_name":"Segmentation"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"},{"task_slug":"unsupervised-object-localization","task_name":"Unsupervised Object Localization"},{"task_slug":"zero-shot-learning","task_name":"Zero-Shot Learning"},{"task_slug":"zero-shot-semantic-segmentation","task_name":"Zero-Shot Semantic Segmentation"},{"task_slug":null,"task_name":"zero-shot-classification"}],"methods":[{"method_slug":"clip","method_name":"CLIP"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2309.14289","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2309.14289"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/wysoczanska/clip-diy","reach":{"status":"ok"}}],"summary":{"ran":7},"by_repo_kind":{"official":{"samples":7,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":7,"samples":[{"code_sha256_prefix":"f78878da5088b102","entry":"collate_fn_masks","repo":"wysoczanska/clip-diy","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/wysoczanska/clip-diy/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f78878da5088b102"}},{"code_sha256_prefix":"9940b696d7ad7c1b","entry":"collate_fn_masks_and_sal","repo":"wysoczanska/clip-diy","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/wysoczanska/clip-diy/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9940b696d7ad7c1b"}},{"code_sha256_prefix":"0f14ddfda99d2e48","entry":"combine_predictions","repo":"wysoczanska/clip-diy","repo_kind":"official","path":"src/semseg/evaluate.py","file_url":"https://github.com/wysoczanska/clip-diy/blob/HEAD/src/semseg/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0f14ddfda99d2e48"}},{"code_sha256_prefix":"0f16729a1c77ac05","entry":"compute_predictions","repo":"wysoczanska/clip-diy","repo_kind":"official","path":"src/semseg/evaluate.py","file_url":"https://github.com/wysoczanska/clip-diy/blob/HEAD/src/semseg/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0f16729a1c77ac05"}},{"code_sha256_prefix":"2b89ae29e59412c8","entry":"get_iou","repo":"wysoczanska/clip-diy","repo_kind":"official","path":"thirdparty/utils.py","file_url":"https://github.com/wysoczanska/clip-diy/blob/HEAD/thirdparty/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2b89ae29e59412c8"}},{"code_sha256_prefix":"286d7d2d6ee7ce58","entry":"has_batchnorms","repo":"wysoczanska/clip-diy","repo_kind":"official","path":"thirdparty/utils.py","file_url":"https://github.com/wysoczanska/clip-diy/blob/HEAD/thirdparty/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"286d7d2d6ee7ce58"}},{"code_sha256_prefix":"fc83d0189940cb2f","entry":"tqdm_joblib","repo":"wysoczanska/clip-diy","repo_kind":"official","path":"thirdparty/utils.py","file_url":"https://github.com/wysoczanska/clip-diy/blob/HEAD/thirdparty/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fc83d0189940cb2f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}