{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/complementary-random-masking-for-rgb-thermal","title":"Complementary Random Masking for RGB-Thermal Semantic Segmentation","arxiv_id":"2303.17386","date":"2023-03-30","proceeding":null,"authors":["Ukcheol Shin","Kyunghyun Lee","In So Kweon","Jean Oh"],"abstract":"RGB-thermal semantic segmentation is one potential solution to achieve reliable semantic scene understanding in adverse weather and lighting conditions. However, the previous studies mostly focus on designing a multi-modal fusion module without consideration of the nature of multi-modality inputs. Therefore, the networks easily become over-reliant on a single modality, making it difficult to learn complementary and meaningful representations for each modality. This paper proposes 1) a complementary random masking strategy of RGB-T images and 2) self-distillation loss between clean and masked input modalities. The proposed masking strategy prevents over-reliance on a single modality. It also improves the accuracy and robustness of the neural network by forcing the network to segment and classify objects even when one modality is partially available. Also, the proposed self-distillation loss encourages the network to extract complementary and meaningful representations from a single modality or complementary masked modalities. Based on the proposed method, we achieve state-of-the-art performance over three RGB-T semantic segmentation benchmarks. Our source code is available at https://github.com/UkcheolShin/CRM_RGBTSeg.","url_abs":"https://arxiv.org/abs/2303.17386v2","url_pdf":"https://arxiv.org/pdf/2303.17386v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"complementary-random-masking-for-rgb-thermal","repo_url":"https://github.com/UkcheolShin/CRM_RGBTSeg","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"scene-understanding","task_name":"Scene Understanding"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"},{"task_slug":"thermal-image-segmentation","task_name":"Thermal Image Segmentation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/thermal-image-segmentation-on-kp-day-night","task":"Thermal Image Segmentation","dataset":"KP day-night","model":"CRM_RGBTSeg","rank_in_archive_order":2,"of":5,"metrics":{"mIoU":"55.2"},"uses_additional_data":false},{"leaderboard":"/sota/thermal-image-segmentation-on-mfn-dataset","task":"Thermal Image Segmentation","dataset":"MFN Dataset","model":"CRM_RGBT_Seg","rank_in_archive_order":3,"of":55,"metrics":{"mIOU":"61.4"},"uses_additional_data":false},{"leaderboard":"/sota/thermal-image-segmentation-on-pst900","task":"Thermal Image Segmentation","dataset":"PST900","model":"CRM_RGBTSeg","rank_in_archive_order":3,"of":22,"metrics":{"mIoU":"88"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2303.17386","atlas_url":"https://app.syntology.ai/?focus=2303.17386","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.17386"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/UkcheolShin/CRM_RGBTSeg","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":5},"by_repo_kind":{"official":{"samples":5,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"bc2cb481a75c370d","entry":"batch_dice_loss","repo":"UkcheolShin/CRM_RGBTSeg","repo_kind":"official","path":"models/mask2former/matcher.py","file_url":"https://github.com/UkcheolShin/CRM_RGBTSeg/blob/HEAD/models/mask2former/matcher.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bc2cb481a75c370d"}},{"code_sha256_prefix":"1edd24985036b0bf","entry":"batch_sigmoid_ce_loss","repo":"UkcheolShin/CRM_RGBTSeg","repo_kind":"official","path":"models/mask2former/matcher.py","file_url":"https://github.com/UkcheolShin/CRM_RGBTSeg/blob/HEAD/models/mask2former/matcher.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1edd24985036b0bf"}},{"code_sha256_prefix":"2dcb8123d89bb1ff","entry":"calculate_uncertainty","repo":"UkcheolShin/CRM_RGBTSeg","repo_kind":"official","path":"models/mask2former/criterion.py","file_url":"https://github.com/UkcheolShin/CRM_RGBTSeg/blob/HEAD/models/mask2former/criterion.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2dcb8123d89bb1ff"}},{"code_sha256_prefix":"89f75e54ff128be0","entry":"dice_loss","repo":"UkcheolShin/CRM_RGBTSeg","repo_kind":"official","path":"models/mask2former/criterion.py","file_url":"https://github.com/UkcheolShin/CRM_RGBTSeg/blob/HEAD/models/mask2former/criterion.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"89f75e54ff128be0"}},{"code_sha256_prefix":"d0c61e8dba511aa3","entry":"sigmoid_ce_loss","repo":"UkcheolShin/CRM_RGBTSeg","repo_kind":"official","path":"models/mask2former/criterion.py","file_url":"https://github.com/UkcheolShin/CRM_RGBTSeg/blob/HEAD/models/mask2former/criterion.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d0c61e8dba511aa3"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}