{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/oneformer-one-transformer-to-rule-universal","title":"OneFormer: One Transformer to Rule Universal Image Segmentation","arxiv_id":"2211.06220","date":"2022-11-10","proceeding":"CVPR 2023 1","authors":["Jitesh Jain","Jiachen Li","MangTik Chiu","Ali Hassani","Nikita Orlov","Humphrey Shi"],"abstract":"Universal Image Segmentation is not a new concept. Past attempts to unify image segmentation in the last decades include scene parsing, panoptic segmentation, and, more recently, new panoptic architectures. However, such panoptic architectures do not truly unify image segmentation because they need to be trained individually on the semantic, instance, or panoptic segmentation to achieve the best performance. Ideally, a truly universal framework should be trained only once and achieve SOTA performance across all three image segmentation tasks. To that end, we propose OneFormer, a universal image segmentation framework that unifies segmentation with a multi-task train-once design. We first propose a task-conditioned joint training strategy that enables training on ground truths of each domain (semantic, instance, and panoptic segmentation) within a single multi-task training process. Secondly, we introduce a task token to condition our model on the task at hand, making our model task-dynamic to support multi-task training and inference. Thirdly, we propose using a query-text contrastive loss during training to establish better inter-task and inter-class distinctions. Notably, our single OneFormer model outperforms specialized Mask2Former models across all three segmentation tasks on ADE20k, CityScapes, and COCO, despite the latter being trained on each of the three tasks individually with three times the resources. With new ConvNeXt and DiNAT backbones, we observe even more performance improvement. We believe OneFormer is a significant step towards making image segmentation more universal and accessible. To support further research, we open-source our code and models at https://github.com/SHI-Labs/OneFormer","url_abs":"https://arxiv.org/abs/2211.06220v2","url_pdf":"https://arxiv.org/pdf/2211.06220v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"oneformer-one-transformer-to-rule-universal","repo_url":"https://github.com/SHI-Labs/OneFormer","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"oneformer-one-transformer-to-rule-universal","repo_url":"https://github.com/huggingface/transformers","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"oneformer-one-transformer-to-rule-universal","repo_url":"https://github.com/MindCode-4/code-2/tree/main/oneformer","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"oneformer-one-transformer-to-rule-universal","repo_url":"https://github.com/yangyucheng000/University/tree/main/model-1/oneformer","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null}],"tasks":[{"task_slug":"instance-segmentation","task_name":"Instance Segmentation"},{"task_slug":"panoptic-segmentation","task_name":"Panoptic Segmentation"},{"task_slug":"scene-parsing","task_name":"Scene Parsing"},{"task_slug":"segmentation","task_name":"Segmentation"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"},{"task_slug":"universal-segmentation","task_name":"Universal Segmentation"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"convnext","method_name":"ConvNeXt"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"na","method_name":"Neighborhood Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/instance-segmentation-on-ade20k-val","task":"Instance Segmentation","dataset":"ADE20K val","model":"OneFormer (InternImage-H, emb_dim=1024, single-scale, 896x896, COCO-Pretrained)","rank_in_archive_order":1,"of":14,"metrics":{"AP":"44.2","APL":"64.3","APM":"49.9","APS":"23.7"},"uses_additional_data":false},{"leaderboard":"/sota/instance-segmentation-on-ade20k-val","task":"Instance Segmentation","dataset":"ADE20K val","model":"OneFormer (DiNAT-L, single-scale, 1280x1280, COCO-pretrain)","rank_in_archive_order":4,"of":14,"metrics":{"AP":"40.2","APL":"59.7","APM":"44.4","APS":"19.2"},"uses_additional_data":true},{"leaderboard":"/sota/instance-segmentation-on-ade20k-val","task":"Instance Segmentation","dataset":"ADE20K val","model":"OneFormer (DiNAT-L, single-scale)","rank_in_archive_order":7,"of":14,"metrics":{"AP":"36.0"},"uses_additional_data":false},{"leaderboard":"/sota/instance-segmentation-on-ade20k-val","task":"Instance Segmentation","dataset":"ADE20K val","model":"OneFormer (Swin-L, single-scale)","rank_in_archive_order":8,"of":14,"metrics":{"AP":"35.9"},"uses_additional_data":false},{"leaderboard":"/sota/instance-segmentation-on-coco-val-panoptic","task":"Instance Segmentation","dataset":"COCO val (panoptic labels)","model":"OneFormer (InternImage-H, emb_dim=1024, single-scale)","rank_in_archive_order":1,"of":4,"metrics":{"AP":"52.0"},"uses_additional_data":false},{"leaderboard":"/sota/instance-segmentation-on-coco-val-panoptic","task":"Instance Segmentation","dataset":"COCO val (panoptic labels)","model":"OneFormer (DiNAT-L, single-scale)","rank_in_archive_order":2,"of":4,"metrics":{"AP":"49.2"},"uses_additional_data":false},{"leaderboard":"/sota/instance-segmentation-on-coco-val-panoptic","task":"Instance Segmentation","dataset":"COCO val (panoptic labels)","model":"OneFormer (Swin-L, single-scale)","rank_in_archive_order":4,"of":4,"metrics":{"AP":"49.0"},"uses_additional_data":false},{"leaderboard":"/sota/instance-segmentation-on-cityscapes-val","task":"Instance Segmentation","dataset":"Cityscapes val","model":"OneFormer (ConvNeXt-L, single-scale, Mapillary-Pretrained)","rank_in_archive_order":2,"of":17,"metrics":{"mask AP":"48.7"},"uses_additional_data":true},{"leaderboard":"/sota/instance-segmentation-on-cityscapes-val","task":"Instance Segmentation","dataset":"Cityscapes val","model":"OneFormer (DiNAT-L, single-scale)","rank_in_archive_order":5,"of":17,"metrics":{"mask AP":"45.6"},"uses_additional_data":false},{"leaderboard":"/sota/instance-segmentation-on-cityscapes-val","task":"Instance Segmentation","dataset":"Cityscapes val","model":"OneFormer (Swin-L, single-scale)","rank_in_archive_order":6,"of":17,"metrics":{"mask AP":"45.6"},"uses_additional_data":false},{"leaderboard":"/sota/panoptic-segmentation-on-ade20k-val","task":"Panoptic Segmentation","dataset":"ADE20K val","model":"OneFormer (InternImage-H, emb_dim=256, single-scale, 896x896)","rank_in_archive_order":1,"of":25,"metrics":{"AP":"40.2","PQ":"54.5","mIoU":"60.4"},"uses_additional_data":false},{"leaderboard":"/sota/panoptic-segmentation-on-ade20k-val","task":"Panoptic Segmentation","dataset":"ADE20K val","model":"OneFormer (DiNAT-L, single-scale, 1280x1280, COCO-Pretrain)","rank_in_archive_order":4,"of":25,"metrics":{"PQ":"53.4","mIoU":"58.9"},"uses_additional_data":true},{"leaderboard":"/sota/panoptic-segmentation-on-ade20k-val","task":"Panoptic Segmentation","dataset":"ADE20K val","model":"OneFormer (DiNAT-L, single-scale, 1280x1280)","rank_in_archive_order":8,"of":25,"metrics":{"AP":"37.1","PQ":"51.5","mIoU":"58.3"},"uses_additional_data":false},{"leaderboard":"/sota/panoptic-segmentation-on-ade20k-val","task":"Panoptic Segmentation","dataset":"ADE20K val","model":"OneFormer (Swin-L, single-scale, 1280x1280)","rank_in_archive_order":9,"of":25,"metrics":{"AP":"37.8","PQ":"51.4","mIoU":"57.0"},"uses_additional_data":false},{"leaderboard":"/sota/panoptic-segmentation-on-ade20k-val","task":"Panoptic Segmentation","dataset":"ADE20K val","model":"OneFormer (DiNAT-L, single-scale, 640x640)","rank_in_archive_order":11,"of":25,"metrics":{"AP":"36.0","PQ":"50.5","mIoU":"58.3"},"uses_additional_data":false},{"leaderboard":"/sota/panoptic-segmentation-on-ade20k-val","task":"Panoptic Segmentation","dataset":"ADE20K val","model":"OneFormer (ConvNeXt-XL, single-scale, 640x640)","rank_in_archive_order":12,"of":25,"metrics":{"AP":"36.3","PQ":"50.1","mIoU":"57.4"},"uses_additional_data":false},{"leaderboard":"/sota/panoptic-segmentation-on-ade20k-val","task":"Panoptic Segmentation","dataset":"ADE20K val","model":"OneFormer (ConvNeXt-L, single-scale, 640x640)","rank_in_archive_order":13,"of":25,"metrics":{"AP":"36.2","PQ":"50.0","mIoU":"56.6"},"uses_additional_data":false},{"leaderboard":"/sota/panoptic-segmentation-on-ade20k-val","task":"Panoptic Segmentation","dataset":"ADE20K val","model":"OneFormer (Swin-L, single-scale, 640x640)","rank_in_archive_order":14,"of":25,"metrics":{"AP":"35.9","PQ":"49.8","mIoU":"57.0"},"uses_additional_data":false},{"leaderboard":"/sota/panoptic-segmentation-on-coco-minival","task":"Panoptic Segmentation","dataset":"COCO minival","model":"OneFormer (InternImage-H,single-scale)","rank_in_archive_order":2,"of":31,"metrics":{"AP":"52.0","PQ":"60.0","PQst":"49.2","PQth":"67.1","mIoU":"68.8"},"uses_additional_data":false},{"leaderboard":"/sota/panoptic-segmentation-on-coco-minival","task":"Panoptic Segmentation","dataset":"COCO minival","model":"OneFormer (DiNAT-L, single-scale)","rank_in_archive_order":14,"of":31,"metrics":{"AP":"49.2","PQ":"58.0","PQst":"48.4","PQth":"64.3","mIoU":"68.1"},"uses_additional_data":false},{"leaderboard":"/sota/panoptic-segmentation-on-coco-minival","task":"Panoptic Segmentation","dataset":"COCO minival","model":"OneFormer (Swin-L, single-scale)","rank_in_archive_order":16,"of":31,"metrics":{"AP":"49.0","PQ":"57.9","PQst":"48.0","PQth":"64.4","mIoU":"67.4"},"uses_additional_data":false},{"leaderboard":"/sota/panoptic-segmentation-on-cityscapes-test","task":"Panoptic Segmentation","dataset":"Cityscapes test","model":"OneFormer (ConvNeXt-L, single-scale, Mapillary Vistas-Pretrained)","rank_in_archive_order":1,"of":10,"metrics":{"PQ":"68.0"},"uses_additional_data":true},{"leaderboard":"/sota/panoptic-segmentation-on-cityscapes-val","task":"Panoptic Segmentation","dataset":"Cityscapes val","model":"OneFormer (ConvNeXt-L, single-scale, 512x1024, Mapillary Vistas-pretrained)","rank_in_archive_order":2,"of":37,"metrics":{"AP":"48.7","PQ":"70.1","PQst":"74.1","PQth":"64.6","mIoU":"84.6"},"uses_additional_data":true},{"leaderboard":"/sota/panoptic-segmentation-on-cityscapes-val","task":"Panoptic Segmentation","dataset":"Cityscapes val","model":"OneFormer (ConvNeXt-L, single-scale)","rank_in_archive_order":4,"of":37,"metrics":{"AP":"46.5","PQ":"68.51","mIoU":"83.0"},"uses_additional_data":false},{"leaderboard":"/sota/panoptic-segmentation-on-cityscapes-val","task":"Panoptic Segmentation","dataset":"Cityscapes val","model":"OneFormer (ConvNeXt-XL, single-scale)","rank_in_archive_order":7,"of":37,"metrics":{"AP":"46.7","PQ":"68.4","mIoU":"83.6"},"uses_additional_data":false},{"leaderboard":"/sota/panoptic-segmentation-on-cityscapes-val","task":"Panoptic Segmentation","dataset":"Cityscapes val","model":"OneFormer (DiNAT-L, single-scale)","rank_in_archive_order":10,"of":37,"metrics":{"AP":"45.6","PQ":"67.6","mIoU":"83.1"},"uses_additional_data":false},{"leaderboard":"/sota/panoptic-segmentation-on-cityscapes-val","task":"Panoptic Segmentation","dataset":"Cityscapes val","model":"OneFormer (Swin-L, single-scale)","rank_in_archive_order":13,"of":37,"metrics":{"AP":"45.6","PQ":"67.2","mIoU":"83.0"},"uses_additional_data":false},{"leaderboard":"/sota/panoptic-segmentation-on-mapillary-val","task":"Panoptic Segmentation","dataset":"Mapillary val","model":"OneFormer (DiNAT-L, single-scale)","rank_in_archive_order":1,"of":13,"metrics":{"PQ":"46.7","PQst":"54.9","PQth":"40.5","mIoU":"61.7"},"uses_additional_data":false},{"leaderboard":"/sota/panoptic-segmentation-on-mapillary-val","task":"Panoptic Segmentation","dataset":"Mapillary val","model":"OneFormer (ConvNeXt-L, single-scale)","rank_in_archive_order":2,"of":13,"metrics":{"PQ":"46.4","PQst":"54.0","PQth":"40.6","mIoU":"61.6"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-ade20k-val","task":"Semantic Segmentation","dataset":"ADE20K val","model":"OneFormer (InternImage-H, emb_dim=256, multi-scale, 896x896)","rank_in_archive_order":6,"of":95,"metrics":{"mIoU":"60.8"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-ade20k-val","task":"Semantic Segmentation","dataset":"ADE20K val","model":"OneFormer (DiNAT-L, multi-scale, 896x896)","rank_in_archive_order":9,"of":95,"metrics":{"mIoU":"58.6"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-ade20k-val","task":"Semantic Segmentation","dataset":"ADE20K val","model":"OneFormer (DiNAT-L, multi-scale, 640x640)","rank_in_archive_order":11,"of":95,"metrics":{"mIoU":"58.4"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-ade20k-val","task":"Semantic Segmentation","dataset":"ADE20K val","model":"OneFormer (Swin-L, multi-scale, 896x896)","rank_in_archive_order":14,"of":95,"metrics":{"mIoU":"58.3"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-ade20k-val","task":"Semantic Segmentation","dataset":"ADE20K val","model":"OneFormer (Swin-L, multi-scale, 640x640)","rank_in_archive_order":19,"of":95,"metrics":{"mIoU":"57.7"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-coco-1","task":"Semantic Segmentation","dataset":"COCO (Common Objects in Context)","model":"OneFormer (InternImage-H, emb_dim=1024, single-scale)","rank_in_archive_order":3,"of":9,"metrics":{"mIoU":"68.8"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-coco-1","task":"Semantic Segmentation","dataset":"COCO (Common Objects in Context)","model":"OneFormer (DiNAT-L, single-scale)","rank_in_archive_order":5,"of":9,"metrics":{"mIoU":"68.1"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-coco-1","task":"Semantic Segmentation","dataset":"COCO (Common Objects in Context)","model":"OneFormer (Swin-L, single-scale)","rank_in_archive_order":6,"of":9,"metrics":{"mIoU":"67.4"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-cityscapes-val","task":"Semantic Segmentation","dataset":"Cityscapes val","model":"OneFormer (ConvNeXt-XL, Mapillary, multi-scale)","rank_in_archive_order":9,"of":99,"metrics":{"mIoU":"85.8"},"uses_additional_data":true},{"leaderboard":"/sota/semantic-segmentation-on-cityscapes-val","task":"Semantic Segmentation","dataset":"Cityscapes val","model":"OneFormer (ConvNeXt-XL, multi-scale)","rank_in_archive_order":14,"of":99,"metrics":{"mIoU":"84.6"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-cityscapes-val","task":"Semantic Segmentation","dataset":"Cityscapes val","model":"OneFormer (Swin-L, multi-scale)","rank_in_archive_order":16,"of":99,"metrics":{"mIoU":"84.4"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-mapillary-val","task":"Semantic Segmentation","dataset":"Mapillary val","model":"OneFormer (DiNAT-L, multi-scale)","rank_in_archive_order":2,"of":8,"metrics":{"mIoU":"64.9"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2211.06220","atlas_url":"https://app.syntology.ai/?focus=2211.06220","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.06220"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/huggingface/transformers","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/SHI-Labs/OneFormer","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/MindCode-4/code-2/tree/main/oneformer","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yangyucheng000/University/tree/main/model-1/oneformer","reach":null}],"summary":{"ran":2,"unverified":3},"by_repo_kind":{"official":{"samples":5,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"0b93a4545ba92b2d","entry":"linear_sum_assignment_with_nan","repo":"SHI-Labs/OneFormer","repo_kind":"official","path":"oneformer/modeling/matcher.py","file_url":"https://github.com/SHI-Labs/OneFormer/blob/HEAD/oneformer/modeling/matcher.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0b93a4545ba92b2d"}},{"code_sha256_prefix":"dc4a40c394479bd8","entry":"sigmoid_ce_loss","repo":"SHI-Labs/OneFormer","repo_kind":"official","path":"oneformer/modeling/criterion.py","file_url":"https://github.com/SHI-Labs/OneFormer/blob/HEAD/oneformer/modeling/criterion.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dc4a40c394479bd8"}},{"code_sha256_prefix":"bc2cb481a75c370d","entry":"batch_dice_loss","repo":"SHI-Labs/OneFormer","repo_kind":"official","path":"oneformer/modeling/matcher.py","file_url":"https://github.com/SHI-Labs/OneFormer/blob/HEAD/oneformer/modeling/matcher.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bc2cb481a75c370d"}},{"code_sha256_prefix":"1edd24985036b0bf","entry":"batch_sigmoid_ce_loss","repo":"SHI-Labs/OneFormer","repo_kind":"official","path":"oneformer/modeling/matcher.py","file_url":"https://github.com/SHI-Labs/OneFormer/blob/HEAD/oneformer/modeling/matcher.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1edd24985036b0bf"}},{"code_sha256_prefix":"89f75e54ff128be0","entry":"dice_loss","repo":"SHI-Labs/OneFormer","repo_kind":"official","path":"oneformer/modeling/criterion.py","file_url":"https://github.com/SHI-Labs/OneFormer/blob/HEAD/oneformer/modeling/criterion.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"89f75e54ff128be0"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}