{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/mask3d-for-3d-semantic-instance-segmentation","title":"Mask3D: Mask Transformer for 3D Semantic Instance Segmentation","arxiv_id":"2210.03105","date":"2022-10-06","proceeding":null,"authors":["Jonas Schult","Francis Engelmann","Alexander Hermans","Or Litany","Siyu Tang","Bastian Leibe"],"abstract":"Modern 3D semantic instance segmentation approaches predominantly rely on specialized voting mechanisms followed by carefully designed geometric clustering techniques. Building on the successes of recent Transformer-based methods for object detection and image segmentation, we propose the first Transformer-based approach for 3D semantic instance segmentation. We show that we can leverage generic Transformer building blocks to directly predict instance masks from 3D point clouds. In our model called Mask3D each object instance is represented as an instance query. Using Transformer decoders, the instance queries are learned by iteratively attending to point cloud features at multiple scales. Combined with point features, the instance queries directly yield all instance masks in parallel. Mask3D has several advantages over current state-of-the-art approaches, since it neither relies on (1) voting schemes which require hand-selected geometric properties (such as centers) nor (2) geometric grouping mechanisms requiring manually-tuned hyper-parameters (e.g. radii) and (3) enables a loss that directly optimizes instance masks. Mask3D sets a new state-of-the-art on ScanNet test (+6.2 mAP), S3DIS 6-fold (+10.1 mAP), STPLS3D (+11.2 mAP) and ScanNet200 test (+12.4 mAP).","url_abs":"https://arxiv.org/abs/2210.03105v2","url_pdf":"https://arxiv.org/pdf/2210.03105v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"mask3d-for-3d-semantic-instance-segmentation","repo_url":"https://github.com/jonasschult/mask3d","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"3d-instance-segmentation-1","task_name":"3D Instance Segmentation"},{"task_slug":"3d-semantic-instance-segmentation","task_name":"3D Semantic Instance Segmentation"},{"task_slug":"instance-segmentation","task_name":"Instance Segmentation"},{"task_slug":"segmentation","task_name":"Segmentation"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"test","method_name":"Test"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/3d-instance-segmentation-on-s3dis","task":"3D Instance Segmentation","dataset":"S3DIS","model":"Mask3D","rank_in_archive_order":2,"of":21,"metrics":{"AP@50":"75.5","mAP":"64.5"},"uses_additional_data":false},{"leaderboard":"/sota/3d-instance-segmentation-on-stpls3d","task":"3D Instance Segmentation","dataset":"STPLS3D","model":"Mask3D","rank_in_archive_order":2,"of":9,"metrics":{"AP":"57.3","AP25":"81.6","AP50":"74.3"},"uses_additional_data":false},{"leaderboard":"/sota/3d-instance-segmentation-on-scannetv2","task":"3D Instance Segmentation","dataset":"ScanNet(v2)","model":"Mask3D","rank_in_archive_order":7,"of":32,"metrics":{"mAP":"55.2","mAP @ 50":"78.0","mAP@25":"87.0"},"uses_additional_data":false},{"leaderboard":"/sota/3d-instance-segmentation-on-scannet200","task":"3D Instance Segmentation","dataset":"ScanNet200","model":"Mask3D","rank_in_archive_order":2,"of":5,"metrics":{"mAP":"27.8"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-replica","task":"Semantic Segmentation","dataset":"Replica","model":"Mask3D","rank_in_archive_order":3,"of":5,"metrics":{"mIoU":"22.6"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2210.03105","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.03105"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jonasschult/mask3d","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":1,"ran_fixture":1,"unverified":8},"by_repo_kind":{"official":{"samples":10,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"da426b749d2b3e6b","entry":"make_pred_info","repo":"jonasschult/mask3d","repo_kind":"official","path":"benchmark/evaluate_semantic_instance.py","file_url":"https://github.com/jonasschult/mask3d/blob/HEAD/benchmark/evaluate_semantic_instance.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"da426b749d2b3e6b"}},{"code_sha256_prefix":"818099bdffc93897","entry":"shift_scale_points","repo":"jonasschult/mask3d","repo_kind":"official","path":"models/position_embedding.py","file_url":"https://github.com/jonasschult/mask3d/blob/HEAD/models/position_embedding.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"818099bdffc93897"}},{"code_sha256_prefix":"bc2cb481a75c370d","entry":"batch_dice_loss","repo":"jonasschult/mask3d","repo_kind":"official","path":"models/matcher.py","file_url":"https://github.com/jonasschult/mask3d/blob/HEAD/models/matcher.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bc2cb481a75c370d"}},{"code_sha256_prefix":"1edd24985036b0bf","entry":"batch_sigmoid_ce_loss","repo":"jonasschult/mask3d","repo_kind":"official","path":"models/matcher.py","file_url":"https://github.com/jonasschult/mask3d/blob/HEAD/models/matcher.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1edd24985036b0bf"}},{"code_sha256_prefix":"2dcb8123d89bb1ff","entry":"calculate_uncertainty","repo":"jonasschult/mask3d","repo_kind":"official","path":"models/criterion.py","file_url":"https://github.com/jonasschult/mask3d/blob/HEAD/models/criterion.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2dcb8123d89bb1ff"}},{"code_sha256_prefix":"fee7d4228b4a171d","entry":"compute_averages","repo":"jonasschult/mask3d","repo_kind":"official","path":"benchmark/evaluate_semantic_instance.py","file_url":"https://github.com/jonasschult/mask3d/blob/HEAD/benchmark/evaluate_semantic_instance.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fee7d4228b4a171d"}},{"code_sha256_prefix":"89f75e54ff128be0","entry":"dice_loss","repo":"jonasschult/mask3d","repo_kind":"official","path":"models/criterion.py","file_url":"https://github.com/jonasschult/mask3d/blob/HEAD/models/criterion.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"89f75e54ff128be0"}},{"code_sha256_prefix":"6f0bdc00532cb7e6","entry":"evaluate_matches","repo":"jonasschult/mask3d","repo_kind":"official","path":"benchmark/evaluate_semantic_instance.py","file_url":"https://github.com/jonasschult/mask3d/blob/HEAD/benchmark/evaluate_semantic_instance.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6f0bdc00532cb7e6"}},{"code_sha256_prefix":"58cc9ff3bf75e753","entry":"nested_tensor_from_tensor_list","repo":"jonasschult/mask3d","repo_kind":"official","path":"models/misc.py","file_url":"https://github.com/jonasschult/mask3d/blob/HEAD/models/misc.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"58cc9ff3bf75e753"}},{"code_sha256_prefix":"d0c61e8dba511aa3","entry":"sigmoid_ce_loss","repo":"jonasschult/mask3d","repo_kind":"official","path":"models/criterion.py","file_url":"https://github.com/jonasschult/mask3d/blob/HEAD/models/criterion.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d0c61e8dba511aa3"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}