{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/open-vocabulary-segmentation-with-semantic","title":"Open-Vocabulary Segmentation with Semantic-Assisted Calibration","arxiv_id":"2312.04089","date":"2023-12-07","proceeding":"CVPR 2024 1","authors":["Yong liu","Sule Bai","Guanbin Li","Yitong Wang","Yansong Tang"],"abstract":"This paper studies open-vocabulary segmentation (OVS) through calibrating in-vocabulary and domain-biased embedding space with generalized contextual prior of CLIP. As the core of open-vocabulary understanding, alignment of visual content with the semantics of unbounded text has become the bottleneck of this field. To address this challenge, recent works propose to utilize CLIP as an additional classifier and aggregate model predictions with CLIP classification results. Despite their remarkable progress, performance of OVS methods in relevant scenarios is still unsatisfactory compared with supervised counterparts. We attribute this to the in-vocabulary embedding and domain-biased CLIP prediction. To this end, we present a Semantic-assisted CAlibration Network (SCAN). In SCAN, we incorporate generalized semantic prior of CLIP into proposal embedding to avoid collapsing on known categories. Besides, a contextual shift strategy is applied to mitigate the lack of global context and unnatural background noise. With above designs, SCAN achieves state-of-the-art performance on all popular open-vocabulary segmentation benchmarks. Furthermore, we also focus on the problem of existing evaluation system that ignores semantic duplication across categories, and propose a new metric called Semantic-Guided IoU (SG-IoU).","url_abs":"https://arxiv.org/abs/2312.04089v2","url_pdf":"https://arxiv.org/pdf/2312.04089v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"open-vocabulary-segmentation-with-semantic","repo_url":"https://github.com/workforai/scan","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"open-vocabulary-segmentation-with-semantic","repo_url":"https://github.com/yongliu20/SCAN","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}}],"tasks":[{"task_slug":"attribute","task_name":"Attribute"},{"task_slug":"open-vocabulary-semantic-segmentation","task_name":"Open Vocabulary Semantic Segmentation"}],"methods":[{"method_slug":"clip","method_name":"CLIP"},{"method_slug":"focus","method_name":"Focus"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/open-vocabulary-semantic-segmentation-on-2","task":"Open Vocabulary Semantic Segmentation","dataset":"ADE20K-150","model":"SCAN","rank_in_archive_order":12,"of":23,"metrics":{"mIoU":"33.5"},"uses_additional_data":false},{"leaderboard":"/sota/open-vocabulary-semantic-segmentation-on-3","task":"Open Vocabulary Semantic Segmentation","dataset":"ADE20K-847","model":"SCAN","rank_in_archive_order":10,"of":19,"metrics":{"mIoU":"14.0"},"uses_additional_data":false},{"leaderboard":"/sota/open-vocabulary-semantic-segmentation-on-7","task":"Open Vocabulary Semantic Segmentation","dataset":"PASCAL Context-459","model":"SCAN","rank_in_archive_order":11,"of":15,"metrics":{"mIoU":"16.7"},"uses_additional_data":false},{"leaderboard":"/sota/open-vocabulary-semantic-segmentation-on-1","task":"Open Vocabulary Semantic Segmentation","dataset":"PASCAL Context-59","model":"SCAN","rank_in_archive_order":11,"of":24,"metrics":{"mIoU":"59.3"},"uses_additional_data":false},{"leaderboard":"/sota/open-vocabulary-semantic-segmentation-on-5","task":"Open Vocabulary Semantic Segmentation","dataset":"PascalVOC-20","model":"SCAN","rank_in_archive_order":3,"of":20,"metrics":{"mIoU":"97.2"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2312.04089","atlas_url":"https://app.syntology.ai/?focus=2312.04089","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.04089"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/workforai/scan","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yongliu20/SCAN","reach":{"status":"ok","spdx":"NOASSERTION"}}],"summary":{"ran":3,"ran_fixture":1,"ran_draft_wrong":1,"unverified":5},"by_repo_kind":{"official":{"samples":10,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":10,"samples":[{"code_sha256_prefix":"4b40d7fecc852bbe","entry":"crop_with_mask","repo":"workforai/scan","repo_kind":"official","path":"scan/modeling/clip_adapter/utils.py","file_url":"https://github.com/workforai/scan/blob/HEAD/scan/modeling/clip_adapter/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"4b40d7fecc852bbe"}},{"code_sha256_prefix":"2fc0075c2a14b806","entry":"expand_box","repo":"workforai/scan","repo_kind":"official","path":"scan/modeling/clip_adapter/utils.py","file_url":"https://github.com/workforai/scan/blob/HEAD/scan/modeling/clip_adapter/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"2fc0075c2a14b806"}},{"code_sha256_prefix":"50372092a6e364ac","entry":"mask2box","repo":"workforai/scan","repo_kind":"official","path":"scan/modeling/clip_adapter/utils.py","file_url":"https://github.com/workforai/scan/blob/HEAD/scan/modeling/clip_adapter/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"50372092a6e364ac"}},{"code_sha256_prefix":"f9fd6241d935f07b","entry":"window_partition","repo":"workforai/scan","repo_kind":"official","path":"scan/modeling/backbone/swin.py","file_url":"https://github.com/workforai/scan/blob/HEAD/scan/modeling/backbone/swin.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f9fd6241d935f07b"}},{"code_sha256_prefix":"fb32094c6dbece71","entry":"window_reverse","repo":"workforai/scan","repo_kind":"official","path":"scan/modeling/backbone/swin.py","file_url":"https://github.com/workforai/scan/blob/HEAD/scan/modeling/backbone/swin.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fb32094c6dbece71"}},{"code_sha256_prefix":"bc2cb481a75c370d","entry":"batch_dice_loss","repo":"workforai/scan","repo_kind":"official","path":"scan/modeling/matcher.py","file_url":"https://github.com/workforai/scan/blob/HEAD/scan/modeling/matcher.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bc2cb481a75c370d"}},{"code_sha256_prefix":"1edd24985036b0bf","entry":"batch_sigmoid_ce_loss","repo":"workforai/scan","repo_kind":"official","path":"scan/modeling/matcher.py","file_url":"https://github.com/workforai/scan/blob/HEAD/scan/modeling/matcher.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1edd24985036b0bf"}},{"code_sha256_prefix":"2dcb8123d89bb1ff","entry":"calculate_uncertainty","repo":"workforai/scan","repo_kind":"official","path":"scan/modeling/criterion.py","file_url":"https://github.com/workforai/scan/blob/HEAD/scan/modeling/criterion.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2dcb8123d89bb1ff"}},{"code_sha256_prefix":"89f75e54ff128be0","entry":"dice_loss","repo":"workforai/scan","repo_kind":"official","path":"scan/modeling/criterion.py","file_url":"https://github.com/workforai/scan/blob/HEAD/scan/modeling/criterion.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"89f75e54ff128be0"}},{"code_sha256_prefix":"d0c61e8dba511aa3","entry":"sigmoid_ce_loss","repo":"workforai/scan","repo_kind":"official","path":"scan/modeling/criterion.py","file_url":"https://github.com/workforai/scan/blob/HEAD/scan/modeling/criterion.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d0c61e8dba511aa3"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}