{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/delivering-arbitrary-modal-semantic","title":"Delivering Arbitrary-Modal Semantic Segmentation","arxiv_id":"2303.01480","date":"2023-03-02","proceeding":"CVPR 2023 1","authors":["Jiaming Zhang","Ruiping Liu","Hao Shi","Kailun Yang","Simon Reiß","Kunyu Peng","Haodong Fu","Kaiwei Wang","Rainer Stiefelhagen"],"abstract":"Multimodal fusion can make semantic segmentation more robust. However, fusing an arbitrary number of modalities remains underexplored. To delve into this problem, we create the DeLiVER arbitrary-modal segmentation benchmark, covering Depth, LiDAR, multiple Views, Events, and RGB. Aside from this, we provide this dataset in four severe weather conditions as well as five sensor failure cases to exploit modal complementarity and resolve partial outages. To make this possible, we present the arbitrary cross-modal segmentation model CMNeXt. It encompasses a Self-Query Hub (SQ-Hub) designed to extract effective information from any modality for subsequent fusion with the RGB representation and adds only negligible amounts of parameters (~0.01M) per additional modality. On top, to efficiently and flexibly harvest discriminative cues from the auxiliary modalities, we introduce the simple Parallel Pooling Mixer (PPX). With extensive experiments on a total of six benchmarks, our CMNeXt achieves state-of-the-art performance on the DeLiVER, KITTI-360, MFNet, NYU Depth V2, UrbanLF, and MCubeS datasets, allowing to scale from 1 to 81 modalities. On the freshly collected DeLiVER, the quad-modal CMNeXt reaches up to 66.30% in mIoU with a +9.10% gain as compared to the mono-modal baseline. The DeLiVER dataset and our code are at: https://jamycheung.github.io/DELIVER.html.","url_abs":"https://arxiv.org/abs/2303.01480v1","url_pdf":"https://arxiv.org/pdf/2303.01480v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"delivering-arbitrary-modal-semantic","repo_url":"https://github.com/jamycheung/DELIVER","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"segmentation","task_name":"Segmentation"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"},{"task_slug":"thermal-image-segmentation","task_name":"Thermal Image Segmentation"}],"methods":[],"datasets_introduced":[{"slug":"deliver","name":"DELIVER","full_name":""}],"methods_introduced":[],"results":[{"leaderboard":"/sota/semantic-segmentation-on-bjroad","task":"Semantic Segmentation","dataset":"BJRoad","model":"CMNeXt","rank_in_archive_order":1,"of":11,"metrics":{"IoU":"63.22"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-ddd17","task":"Semantic Segmentation","dataset":"DDD17","model":"CMNeXt","rank_in_archive_order":2,"of":9,"metrics":{"mIoU":"72.67"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-deliver-1","task":"Semantic Segmentation","dataset":"DELIVER","model":"CMNeXt (RGB-D-E-LiDAR)","rank_in_archive_order":3,"of":9,"metrics":{"mIoU":"66.30"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-deliver-1","task":"Semantic Segmentation","dataset":"DELIVER","model":"CMNeXt (RGB-D-LiDAR)","rank_in_archive_order":4,"of":9,"metrics":{"mIoU":"65.50"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-deliver-1","task":"Semantic Segmentation","dataset":"DELIVER","model":"CMNeXt (RGB-D-Event)","rank_in_archive_order":5,"of":9,"metrics":{"mIoU":"64.44"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-deliver-1","task":"Semantic Segmentation","dataset":"DELIVER","model":"CMNeXt (RGB-Depth)","rank_in_archive_order":6,"of":9,"metrics":{"mIoU":"63.58"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-deliver-1","task":"Semantic Segmentation","dataset":"DELIVER","model":"CMNeXt (RGB-LiDAR)","rank_in_archive_order":7,"of":9,"metrics":{"mIoU":"58.04"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-deliver-1","task":"Semantic Segmentation","dataset":"DELIVER","model":"CMNeXt (RGB-Event)","rank_in_archive_order":8,"of":9,"metrics":{"mIoU":"57.48"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-dsec","task":"Semantic Segmentation","dataset":"DSEC","model":"CMNeXt","rank_in_archive_order":2,"of":9,"metrics":{"mIoU":"72.54"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-deliver","task":"Semantic Segmentation","dataset":"DeLiVER","model":"CMNeXt (RGB-D-E-LiDAR)","rank_in_archive_order":5,"of":26,"metrics":{"mIoU":"66.30"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-kitti-360","task":"Semantic Segmentation","dataset":"KITTI-360","model":"CMNeXt (RGB-D-E-LiDAR)","rank_in_archive_order":2,"of":17,"metrics":{"mIoU":"67.84"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-mcubes","task":"Semantic Segmentation","dataset":"MCubeS","model":"CMNeXt (B2 RGB-A-D-N)","rank_in_archive_order":10,"of":22,"metrics":{"mIoU":"51.54%"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-mcubes","task":"Semantic Segmentation","dataset":"MCubeS","model":"CMNeXt (B2 RGB-A-D)","rank_in_archive_order":17,"of":22,"metrics":{"mIoU":"49.48%"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-mcubes","task":"Semantic Segmentation","dataset":"MCubeS","model":"CMNeXt (B2 RGB-A)","rank_in_archive_order":18,"of":22,"metrics":{"mIoU":"48.42%"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-mcubes-p","task":"Semantic Segmentation","dataset":"MCubeS (P)","model":"CMNeXt (B2 RGB-A-D)","rank_in_archive_order":7,"of":8,"metrics":{"mIoU":"49.48"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-mcubes-p","task":"Semantic Segmentation","dataset":"MCubeS (P)","model":"CMNeXt (B2 RGB-A)","rank_in_archive_order":8,"of":8,"metrics":{"mIoU":"48.42"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-nyu-depth-v2","task":"Semantic Segmentation","dataset":"NYU Depth v2","model":"CMNeXt (B4)","rank_in_archive_order":18,"of":121,"metrics":{"Mean IoU":"56.9%"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-porto","task":"Semantic Segmentation","dataset":"Porto","model":"CMNeXt","rank_in_archive_order":1,"of":6,"metrics":{"IoU":"73.12"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-tlcgis","task":"Semantic Segmentation","dataset":"TLCGIS","model":"CMNeXt","rank_in_archive_order":4,"of":6,"metrics":{"IoU":"82.26"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-urbanlf","task":"Semantic Segmentation","dataset":"UrbanLF","model":"CMNeXt (RGB-LF80)","rank_in_archive_order":1,"of":14,"metrics":{"mIoU (Real)":"83.11","mIoU (Syn)":"81.02"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-urbanlf","task":"Semantic Segmentation","dataset":"UrbanLF","model":"CMNeXt (RGB-LF33)","rank_in_archive_order":2,"of":14,"metrics":{"mIoU (Real)":"82.62","mIoU (Syn)":"80.98"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-urbanlf","task":"Semantic Segmentation","dataset":"UrbanLF","model":"CMNeXt (RGB-LF8)","rank_in_archive_order":3,"of":14,"metrics":{"mIoU (Real)":"83.22","mIoU (Syn)":"80.74"},"uses_additional_data":false},{"leaderboard":"/sota/thermal-image-segmentation-on-mfn-dataset","task":"Thermal Image Segmentation","dataset":"MFN Dataset","model":"CMNeXt (B4)","rank_in_archive_order":6,"of":55,"metrics":{"mIOU":"59.9"},"uses_additional_data":false},{"leaderboard":"/sota/thermal-image-segmentation-on-noisy-rs-rgb-t","task":"Thermal Image Segmentation","dataset":"Noisy RS RGB-T Dataset","model":"CMNeXt (B4)","rank_in_archive_order":1,"of":6,"metrics":{"mIoU":"60.3"},"uses_additional_data":false}],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2303.01480","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.01480"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jamycheung/DELIVER","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":6,"unverified":1},"by_repo_kind":{"official":{"samples":7,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"624e13fbf125c2a5","entry":"get_optimizer","repo":"jamycheung/DELIVER","repo_kind":"official","path":"semseg/optimizers.py","file_url":"https://github.com/jamycheung/DELIVER/blob/HEAD/semseg/optimizers.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"624e13fbf125c2a5"}},{"code_sha256_prefix":"1190c7468e2cdaa7","entry":"get_scheduler","repo":"jamycheung/DELIVER","repo_kind":"official","path":"semseg/schedulers.py","file_url":"https://github.com/jamycheung/DELIVER/blob/HEAD/semseg/schedulers.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1190c7468e2cdaa7"}},{"code_sha256_prefix":"36844e88d38a0881","entry":"get_train_augmentation","repo":"jamycheung/DELIVER","repo_kind":"official","path":"semseg/augmentations.py","file_url":"https://github.com/jamycheung/DELIVER/blob/HEAD/semseg/augmentations.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"36844e88d38a0881"}},{"code_sha256_prefix":"1ce37e1fa8008660","entry":"get_train_augmentation","repo":"jamycheung/DELIVER","repo_kind":"official","path":"semseg/augmentations_mm.py","file_url":"https://github.com/jamycheung/DELIVER/blob/HEAD/semseg/augmentations_mm.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1ce37e1fa8008660"}},{"code_sha256_prefix":"c63b165d09644f24","entry":"get_val_augmentation","repo":"jamycheung/DELIVER","repo_kind":"official","path":"semseg/augmentations.py","file_url":"https://github.com/jamycheung/DELIVER/blob/HEAD/semseg/augmentations.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c63b165d09644f24"}},{"code_sha256_prefix":"1096caea85dc06d3","entry":"get_val_augmentation","repo":"jamycheung/DELIVER","repo_kind":"official","path":"semseg/augmentations_mm.py","file_url":"https://github.com/jamycheung/DELIVER/blob/HEAD/semseg/augmentations_mm.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1096caea85dc06d3"}},{"code_sha256_prefix":"3029c24a500aea0e","entry":"get_loss","repo":"jamycheung/DELIVER","repo_kind":"official","path":"semseg/losses.py","file_url":"https://github.com/jamycheung/DELIVER/blob/HEAD/semseg/losses.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3029c24a500aea0e"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}