{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/unsupervised-semantic-segmentation-by-2","title":"Unsupervised Semantic Segmentation by Distilling Feature Correspondences","arxiv_id":"2203.08414","date":"2022-03-16","proceeding":"ICLR 2022 4","authors":["Mark Hamilton","Zhoutong Zhang","Bharath Hariharan","Noah Snavely","William T. Freeman"],"abstract":"Unsupervised semantic segmentation aims to discover and localize semantically meaningful categories within image corpora without any form of annotation. To solve this task, algorithms must produce features for every pixel that are both semantically meaningful and compact enough to form distinct clusters. Unlike previous works which achieve this with a single end-to-end framework, we propose to separate feature learning from cluster compactification. Empirically, we show that current unsupervised feature learning frameworks already generate dense features whose correlations are semantically consistent. This observation motivates us to design STEGO ($\\textbf{S}$elf-supervised $\\textbf{T}$ransformer with $\\textbf{E}$nergy-based $\\textbf{G}$raph $\\textbf{O}$ptimization), a novel framework that distills unsupervised features into high-quality discrete semantic labels. At the core of STEGO is a novel contrastive loss function that encourages features to form compact clusters while preserving their relationships across the corpora. STEGO yields a significant improvement over the prior state of the art, on both the CocoStuff ($\\textbf{+14 mIoU}$) and Cityscapes ($\\textbf{+9 mIoU}$) semantic segmentation challenges.","url_abs":"https://arxiv.org/abs/2203.08414v1","url_pdf":"https://arxiv.org/pdf/2203.08414v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"unsupervised-semantic-segmentation-by-2","repo_url":"https://github.com/mhamilton723/STEGO","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"unsupervised-semantic-segmentation-by-2","repo_url":"https://github.com/leggedrobotics/self_supervised_segmentation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"unsupervised-semantic-segmentation-by-2","repo_url":"https://github.com/merantix-momentum/stego-studies","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"form","task_name":"Form"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"},{"task_slug":"unsupervised-semantic-segmentation","task_name":"Unsupervised Semantic Segmentation"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"crf","method_name":"CRF"},{"method_slug":"dino","method_name":"DINO"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"},{"method_slug":"vision-transformer","method_name":"Vision Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/unsupervised-semantic-segmentation-on-coco-7","task":"Unsupervised Semantic Segmentation","dataset":"COCO-Stuff-27","model":"STEGO (ViT-B/8)","rank_in_archive_order":8,"of":29,"metrics":{"Clustering [Accuracy]":"56.9","Clustering [mIoU]":"28.2"},"uses_additional_data":false},{"leaderboard":"/sota/unsupervised-semantic-segmentation-on-coco-7","task":"Unsupervised Semantic Segmentation","dataset":"COCO-Stuff-27","model":"STEGO (ViT-S/8)","rank_in_archive_order":17,"of":29,"metrics":{"Clustering [mIoU]":"24.5","Linear Classifier [Accuracy]":"74.4","Linear Classifier [mIoU]":"38.3"},"uses_additional_data":false},{"leaderboard":"/sota/unsupervised-semantic-segmentation-on","task":"Unsupervised Semantic Segmentation","dataset":"Cityscapes test","model":"STEGO","rank_in_archive_order":6,"of":14,"metrics":{"Accuracy":"73.2","mIoU":"21.0"},"uses_additional_data":false},{"leaderboard":"/sota/unsupervised-semantic-segmentation-on-potsdam-1","task":"Unsupervised Semantic Segmentation","dataset":"Potsdam-3","model":"STEGO","rank_in_archive_order":6,"of":8,"metrics":{"Accuracy":"77.0"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2203.08414","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.08414"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/leggedrobotics/self_supervised_segmentation","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mhamilton723/STEGO","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/merantix-momentum/stego-studies","reach":null}],"summary":{"ran":2,"ran_honours":2,"ran_fixture":2,"ran_draft_wrong":2,"unverified":6},"by_repo_kind":{"official":{"samples":3,"ran":3,"repositories":1},"listed":{"samples":11,"ran":5,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"4d25be8ebf17fde1","entry":"bool_flag","repo":"leggedrobotics/self_supervised_segmentation","repo_kind":"listed","path":"stego/backbones/dino/utils.py","file_url":"https://github.com/leggedrobotics/self_supervised_segmentation/blob/HEAD/stego/backbones/dino/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4d25be8ebf17fde1"}},{"code_sha256_prefix":"2b5489053b222e00","entry":"clip_gradients","repo":"leggedrobotics/self_supervised_segmentation","repo_kind":"listed","path":"stego/backbones/dino/utils.py","file_url":"https://github.com/leggedrobotics/self_supervised_segmentation/blob/HEAD/stego/backbones/dino/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2b5489053b222e00"}},{"code_sha256_prefix":"361a6b24f11fc50a","entry":"cosine_scheduler","repo":"leggedrobotics/self_supervised_segmentation","repo_kind":"listed","path":"stego/backbones/dino/utils.py","file_url":"https://github.com/leggedrobotics/self_supervised_segmentation/blob/HEAD/stego/backbones/dino/utils.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"361a6b24f11fc50a"}},{"code_sha256_prefix":"55120f2026b56aa2","entry":"drop_path","repo":"leggedrobotics/self_supervised_segmentation","repo_kind":"listed","path":"stego/backbones/dino/vision_transformer.py","file_url":"https://github.com/leggedrobotics/self_supervised_segmentation/blob/HEAD/stego/backbones/dino/vision_transformer.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"55120f2026b56aa2"}},{"code_sha256_prefix":"d6f15c1eec2a900c","entry":"get_class_labels","repo":"merantix-momentum/stego-studies","repo_kind":"listed","path":"src/train_segmentation.py","file_url":"https://github.com/merantix-momentum/stego-studies/blob/HEAD/src/train_segmentation.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d6f15c1eec2a900c"}},{"code_sha256_prefix":"eb2b27a0c094426d","entry":"one_hot_feats","repo":"mhamilton723/STEGO","repo_kind":"official","path":"src/train_segmentation.py","file_url":"https://github.com/mhamilton723/STEGO/blob/HEAD/src/train_segmentation.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"eb2b27a0c094426d"}},{"code_sha256_prefix":"29fed21d5b05ba11","entry":"resize","repo":"mhamilton723/STEGO","repo_kind":"official","path":"src/train_segmentation.py","file_url":"https://github.com/mhamilton723/STEGO/blob/HEAD/src/train_segmentation.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"29fed21d5b05ba11"}},{"code_sha256_prefix":"205e834237bf794c","entry":"shuffle","repo":"mhamilton723/STEGO","repo_kind":"official","path":"src/train_segmentation.py","file_url":"https://github.com/mhamilton723/STEGO/blob/HEAD/src/train_segmentation.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"205e834237bf794c"}},{"code_sha256_prefix":"dbcfa76b17fae535","entry":"get_backbone","repo":"leggedrobotics/self_supervised_segmentation","repo_kind":"listed","path":"stego/backbones/backbone.py","file_url":"https://github.com/leggedrobotics/self_supervised_segmentation/blob/HEAD/stego/backbones/backbone.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dbcfa76b17fae535"}},{"code_sha256_prefix":"a02d5b8c7dc6802d","entry":"get_nn_file_name","repo":"leggedrobotics/self_supervised_segmentation","repo_kind":"listed","path":"stego/utils.py","file_url":"https://github.com/leggedrobotics/self_supervised_segmentation/blob/HEAD/stego/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a02d5b8c7dc6802d"}},{"code_sha256_prefix":"274d426926a54825","entry":"get_transform","repo":"leggedrobotics/self_supervised_segmentation","repo_kind":"listed","path":"stego/utils.py","file_url":"https://github.com/leggedrobotics/self_supervised_segmentation/blob/HEAD/stego/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"274d426926a54825"}},{"code_sha256_prefix":"b1e60ab85b4a8299","entry":"load_image_to_tensor","repo":"leggedrobotics/self_supervised_segmentation","repo_kind":"listed","path":"stego/utils.py","file_url":"https://github.com/leggedrobotics/self_supervised_segmentation/blob/HEAD/stego/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b1e60ab85b4a8299"}},{"code_sha256_prefix":"afc44a0db78b7846","entry":"vit_small","repo":"leggedrobotics/self_supervised_segmentation","repo_kind":"listed","path":"stego/backbones/dino/vision_transformer.py","file_url":"https://github.com/leggedrobotics/self_supervised_segmentation/blob/HEAD/stego/backbones/dino/vision_transformer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"afc44a0db78b7846"}},{"code_sha256_prefix":"a250f0f34719cf17","entry":"vit_tiny","repo":"leggedrobotics/self_supervised_segmentation","repo_kind":"listed","path":"stego/backbones/dino/vision_transformer.py","file_url":"https://github.com/leggedrobotics/self_supervised_segmentation/blob/HEAD/stego/backbones/dino/vision_transformer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a250f0f34719cf17"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}