{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/segmenter-transformer-for-semantic","title":"Segmenter: Transformer for Semantic Segmentation","arxiv_id":"2105.05633","date":"2021-05-12","proceeding":"ICCV 2021 10","authors":["Robin Strudel","Ricardo Garcia","Ivan Laptev","Cordelia Schmid"],"abstract":"Image segmentation is often ambiguous at the level of individual image patches and requires contextual information to reach label consensus. In this paper we introduce Segmenter, a transformer model for semantic segmentation. In contrast to convolution-based methods, our approach allows to model global context already at the first layer and throughout the network. We build on the recent Vision Transformer (ViT) and extend it to semantic segmentation. To do so, we rely on the output embeddings corresponding to image patches and obtain class labels from these embeddings with a point-wise linear decoder or a mask transformer decoder. We leverage models pre-trained for image classification and show that we can fine-tune them on moderate sized datasets available for semantic segmentation. The linear decoder allows to obtain excellent results already, but the performance can be further improved by a mask transformer generating class masks. We conduct an extensive ablation study to show the impact of the different parameters, in particular the performance is better for large models and small patch sizes. Segmenter attains excellent results for semantic segmentation. It outperforms the state of the art on both ADE20K and Pascal Context datasets and is competitive on Cityscapes.","url_abs":"https://arxiv.org/abs/2105.05633v3","url_pdf":"https://arxiv.org/pdf/2105.05633v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"segmenter-transformer-for-semantic","repo_url":"https://github.com/rstrudel/segmenter","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"segmenter-transformer-for-semantic","repo_url":"https://github.com/EricKani/Segmenter-Based-on-OpenMMLab","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"segmenter-transformer-for-semantic","repo_url":"https://github.com/Expedit-LargeScale-Vision-Transformer/Expedit-Segmenter","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"segmenter-transformer-for-semantic","repo_url":"https://github.com/isaaccorley/segmenter-pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"segmenter-transformer-for-semantic","repo_url":"https://github.com/tue-mps/algm-segmenter","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"segmenter-transformer-for-semantic","repo_url":"https://github.com/tue-mps/cts-segmenter","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"segmenter-transformer-for-semantic","repo_url":"https://github.com/BR-IDL/PaddleViT/tree/main/semantic_segmentation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"paddle","reach":null},{"paper_slug":"segmenter-transformer-for-semantic","repo_url":"https://github.com/PaddlePaddle/PaddleSeg","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"paddle","reach":null}],"tasks":[{"task_slug":"decoder","task_name":"Decoder"},{"task_slug":"image-classification","task_name":"Image Classification"},{"task_slug":"image-segmentation","task_name":"Image Segmentation"},{"task_slug":"segmentation","task_name":"Segmentation"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"},{"task_slug":"thermal-image-segmentation","task_name":"Thermal Image Segmentation"},{"task_slug":"image-classification","task_name":"image-classification"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"convolution","method_name":"Convolution"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"},{"method_slug":"vision-transformer","method_name":"Vision Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/semantic-segmentation-on-ade20k","task":"Semantic Segmentation","dataset":"ADE20K","model":"Seg-L-Mask/16 (MS)","rank_in_archive_order":77,"of":235,"metrics":{"Validation mIoU":"53.63"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-ade20k","task":"Semantic Segmentation","dataset":"ADE20K","model":"Seg-B-Mask/16(MS, ViT-B)","rank_in_archive_order":121,"of":235,"metrics":{"Validation mIoU":"50.0"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-ade20k","task":"Semantic Segmentation","dataset":"ADE20K","model":"Seg-B/8 (MS, ViT-B)","rank_in_archive_order":128,"of":235,"metrics":{"Validation mIoU":"49.61"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-ade20k-val","task":"Semantic Segmentation","dataset":"ADE20K val","model":"Seg-L-Mask/16 (MS, ViT-L)","rank_in_archive_order":38,"of":95,"metrics":{"mIoU":"53.63"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-ade20k-val","task":"Semantic Segmentation","dataset":"ADE20K val","model":"Seg-B-Mask/16 (MS, ViT-B)","rank_in_archive_order":53,"of":95,"metrics":{"mIoU":"50.0"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-ade20k-val","task":"Semantic Segmentation","dataset":"ADE20K val","model":"Seg-B/8 (MS, ViT-B)","rank_in_archive_order":56,"of":95,"metrics":{"Pixel Accuracy":"83.37","mIoU":"49.61"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-pascal-context","task":"Semantic Segmentation","dataset":"PASCAL Context","model":"Seg-L-Mask/16","rank_in_archive_order":16,"of":66,"metrics":{"mIoU":"59.0"},"uses_additional_data":false},{"leaderboard":"/sota/thermal-image-segmentation-on-rgb-t-glass","task":"Thermal Image Segmentation","dataset":"RGB-T-Glass-Segmentation","model":"Segmenter","rank_in_archive_order":17,"of":22,"metrics":{"MAE":"0.072"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2105.05633","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.05633"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/rstrudel/segmenter","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/isaaccorley/segmenter-pytorch","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/PaddlePaddle/PaddleSeg","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/EricKani/Segmenter-Based-on-OpenMMLab","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/tue-mps/cts-segmenter","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/BR-IDL/PaddleViT/tree/main/semantic_segmentation","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/tue-mps/algm-segmenter","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Expedit-LargeScale-Vision-Transformer/Expedit-Segmenter","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"unverified":7},"by_repo_kind":{"listed":{"samples":7,"ran":0,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"aae0d919d8e77e3d","entry":"MaskTransformer","repo":"isaaccorley/segmenter-pytorch","repo_kind":"listed","path":"segmenter/segmenter.py","file_url":"https://github.com/isaaccorley/segmenter-pytorch/blob/HEAD/segmenter/segmenter.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"aae0d919d8e77e3d"}},{"code_sha256_prefix":"61186cac81f07daa","entry":"Segmenter","repo":"isaaccorley/segmenter-pytorch","repo_kind":"listed","path":"segmenter/segmenter.py","file_url":"https://github.com/isaaccorley/segmenter-pytorch/blob/HEAD/segmenter/segmenter.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"61186cac81f07daa"}},{"code_sha256_prefix":"92364877f3ee8bbb","entry":"Upsample","repo":"isaaccorley/segmenter-pytorch","repo_kind":"listed","path":"segmenter/segmenter.py","file_url":"https://github.com/isaaccorley/segmenter-pytorch/blob/HEAD/segmenter/segmenter.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"92364877f3ee8bbb"}},{"code_sha256_prefix":"2cc3699f9a695af9","entry":"accuracy","repo":"tue-mps/cts-segmenter","repo_kind":"listed","path":"segm/metrics.py","file_url":"https://github.com/tue-mps/cts-segmenter/blob/HEAD/segm/metrics.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2cc3699f9a695af9"}},{"code_sha256_prefix":"59c2c15636de4edc","entry":"policy_gt_gen","repo":"tue-mps/cts-segmenter","repo_kind":"listed","path":"policynet/policynet/data_ade20k.py","file_url":"https://github.com/tue-mps/cts-segmenter/blob/HEAD/policynet/policynet/data_ade20k.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"59c2c15636de4edc"}},{"code_sha256_prefix":"4b5dbae992d30d9f","entry":"policy_gt_gen","repo":"tue-mps/cts-segmenter","repo_kind":"listed","path":"policynet/policynet/data_cityscapes.py","file_url":"https://github.com/tue-mps/cts-segmenter/blob/HEAD/policynet/policynet/data_cityscapes.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4b5dbae992d30d9f"}},{"code_sha256_prefix":"22ec93cd607fd71b","entry":"policy_gt_gen","repo":"tue-mps/cts-segmenter","repo_kind":"listed","path":"policynet/policynet/data_pcontext.py","file_url":"https://github.com/tue-mps/cts-segmenter/blob/HEAD/policynet/policynet/data_pcontext.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"22ec93cd607fd71b"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}