{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/otseg-multi-prompt-sinkhorn-attention-for","title":"OTSeg: Multi-prompt Sinkhorn Attention for Zero-Shot Semantic Segmentation","arxiv_id":"2403.14183","date":"2024-03-21","proceeding":null,"authors":["Kwanyoung Kim","Yujin Oh","Jong Chul Ye"],"abstract":"The recent success of CLIP has demonstrated promising results in zero-shot semantic segmentation by transferring muiltimodal knowledge to pixel-level classification. However, leveraging pre-trained CLIP knowledge to closely align text embeddings with pixel embeddings still has limitations in existing approaches. To address this issue, we propose OTSeg, a novel multimodal attention mechanism aimed at enhancing the potential of multiple text prompts for matching associated pixel embeddings. We first propose Multi-Prompts Sinkhorn (MPS) based on the Optimal Transport (OT) algorithm, which leads multiple text prompts to selectively focus on various semantic features within image pixels. Moreover, inspired by the success of Sinkformers in unimodal settings, we introduce the extension of MPS, called Multi-Prompts Sinkhorn Attention (MPSA) , which effectively replaces cross-attention mechanisms within Transformer framework in multimodal settings. Through extensive experiments, we demonstrate that OTSeg achieves state-of-the-art (SOTA) performance with significant gains on Zero-Shot Semantic Segmentation (ZS3) tasks across three benchmark datasets.","url_abs":"https://arxiv.org/abs/2403.14183v2","url_pdf":"https://arxiv.org/pdf/2403.14183v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"otseg-multi-prompt-sinkhorn-attention-for","repo_url":"https://github.com/cubeyoung/OTSeg","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"},{"task_slug":"zero-shot-semantic-segmentation","task_name":"Zero-Shot Semantic Segmentation"}],"methods":[{"method_slug":"align","method_name":"ALIGN"},{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"clip","method_name":"CLIP"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"focus","method_name":"Focus"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/zero-shot-semantic-segmentation-on-coco-stuff","task":"Zero-Shot Semantic Segmentation","dataset":"COCO-Stuff","model":"OTSeg+","rank_in_archive_order":1,"of":15,"metrics":{"Inductive Setting hIoU":"41.5","Transductive Setting hIoU":"49.8"},"uses_additional_data":false},{"leaderboard":"/sota/zero-shot-semantic-segmentation-on-coco-stuff","task":"Zero-Shot Semantic Segmentation","dataset":"COCO-Stuff","model":"OTSeg","rank_in_archive_order":3,"of":15,"metrics":{"Inductive Setting hIoU":"41.4","Transductive Setting hIoU":"49.5"},"uses_additional_data":false},{"leaderboard":"/sota/zero-shot-semantic-segmentation-on-pascal-voc","task":"Zero-Shot Semantic Segmentation","dataset":"PASCAL VOC","model":"OTSeg+","rank_in_archive_order":1,"of":13,"metrics":{"Inductive Setting hIoU":"87.4","Transductive Setting hIoU":"94.4"},"uses_additional_data":false},{"leaderboard":"/sota/zero-shot-semantic-segmentation-on-pascal-voc","task":"Zero-Shot Semantic Segmentation","dataset":"PASCAL VOC","model":"OTSeg","rank_in_archive_order":2,"of":13,"metrics":{"Inductive Setting hIoU":"84.5","Transductive Setting hIoU":"94.2"},"uses_additional_data":false}],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2403.14183","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.14183"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/cubeyoung/OTSeg","reach":{"status":"ok"}}],"summary":{"ran":5,"ran_draft_wrong":3,"ran_honours":1,"unverified":1},"by_repo_kind":{"official":{"samples":10,"ran":9,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":10,"samples":[{"code_sha256_prefix":"85d3df53c42250fe","entry":"Sinkhorn_log_exp_sum","repo":"cubeyoung/OTSeg","repo_kind":"official","path":"models/decode_heads/decode_seg.py","file_url":"https://github.com/cubeyoung/OTSeg/blob/HEAD/models/decode_heads/decode_seg.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"85d3df53c42250fe"}},{"code_sha256_prefix":"98f385d847636a3e","entry":"basic_clean","repo":"cubeyoung/OTSeg","repo_kind":"official","path":"models/segmentor/untils.py","file_url":"https://github.com/cubeyoung/OTSeg/blob/HEAD/models/segmentor/untils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"98f385d847636a3e"}},{"code_sha256_prefix":"864ff812c98727df","entry":"cosine_margin_loss","repo":"cubeyoung/OTSeg","repo_kind":"official","path":"models/losses/criterion.py","file_url":"https://github.com/cubeyoung/OTSeg/blob/HEAD/models/losses/criterion.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"864ff812c98727df"}},{"code_sha256_prefix":"016dba4163da28df","entry":"dice_loss","repo":"cubeyoung/OTSeg","repo_kind":"official","path":"models/losses/criterion.py","file_url":"https://github.com/cubeyoung/OTSeg/blob/HEAD/models/losses/criterion.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"016dba4163da28df"}},{"code_sha256_prefix":"7816c6d65ab55d8f","entry":"get_angles","repo":"cubeyoung/OTSeg","repo_kind":"official","path":"models/decode_heads/utils.py","file_url":"https://github.com/cubeyoung/OTSeg/blob/HEAD/models/decode_heads/utils.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7816c6d65ab55d8f"}},{"code_sha256_prefix":"d919ae32e5e4e616","entry":"get_pairs","repo":"cubeyoung/OTSeg","repo_kind":"official","path":"models/segmentor/untils.py","file_url":"https://github.com/cubeyoung/OTSeg/blob/HEAD/models/segmentor/untils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d919ae32e5e4e616"}},{"code_sha256_prefix":"da3acf7173bd84f0","entry":"positional_encoding","repo":"cubeyoung/OTSeg","repo_kind":"official","path":"models/decode_heads/utils.py","file_url":"https://github.com/cubeyoung/OTSeg/blob/HEAD/models/decode_heads/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"da3acf7173bd84f0"}},{"code_sha256_prefix":"2675e89c94ae08bf","entry":"sigmoid_focal_loss","repo":"cubeyoung/OTSeg","repo_kind":"official","path":"models/losses/criterion.py","file_url":"https://github.com/cubeyoung/OTSeg/blob/HEAD/models/losses/criterion.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2675e89c94ae08bf"}},{"code_sha256_prefix":"9542161e9640b858","entry":"whitespace_clean","repo":"cubeyoung/OTSeg","repo_kind":"official","path":"models/segmentor/untils.py","file_url":"https://github.com/cubeyoung/OTSeg/blob/HEAD/models/segmentor/untils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9542161e9640b858"}},{"code_sha256_prefix":"58cc9ff3bf75e753","entry":"nested_tensor_from_tensor_list","repo":"cubeyoung/OTSeg","repo_kind":"official","path":"models/losses/misc.py","file_url":"https://github.com/cubeyoung/OTSeg/blob/HEAD/models/losses/misc.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"58cc9ff3bf75e753"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}