{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/mmanet-margin-aware-distillation-and-modality","title":"MMANet: Margin-aware Distillation and Modality-aware Regularization for Incomplete Multimodal Learning","arxiv_id":"2304.08028","date":"2023-04-17","proceeding":"CVPR 2023 1","authors":["Shicai Wei","Yang Luo","Chunbo Luo"],"abstract":"Multimodal learning has shown great potentials in numerous scenes and attracts increasing interest recently. However, it often encounters the problem of missing modality data and thus suffers severe performance degradation in practice. To this end, we propose a general framework called MMANet to assist incomplete multimodal learning. It consists of three components: the deployment network used for inference, the teacher network transferring comprehensive multimodal information to the deployment network, and the regularization network guiding the deployment network to balance weak modality combinations. Specifically, we propose a novel margin-aware distillation (MAD) to assist the information transfer by weighing the sample contribution with the classification uncertainty. This encourages the deployment network to focus on the samples near decision boundaries and acquire the refined inter-class margin. Besides, we design a modality-aware regularization (MAR) algorithm to mine the weak modality combinations and guide the regularization network to calculate prediction loss for them. This forces the deployment network to improve its representation ability for the weak modality combinations adaptively. Finally, extensive experiments on multimodal classification and segmentation tasks demonstrate that our MMANet outperforms the state-of-the-art significantly. Code is available at: https://github.com/shicaiwei123/MMANet","url_abs":"https://arxiv.org/abs/2304.08028v1","url_pdf":"https://arxiv.org/pdf/2304.08028v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"mmanet-margin-aware-distillation-and-modality","repo_url":"https://github.com/shicaiwei123/mmanet","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/semantic-segmentation-on-nyu-depth-v2","task":"Semantic Segmentation","dataset":"NYU Depth v2","model":"MMANet","rank_in_archive_order":68,"of":121,"metrics":{"Mean IoU":"49.62%"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2304.08028","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2304.08028"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/shicaiwei123/mmanet","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":14},"by_repo_kind":{"official":{"samples":14,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"cabd997ca3e75a0c","entry":"calc_accuracy","repo":"shicaiwei123/mmanet","repo_kind":"official","path":"classification/lib/model_develop_utils.py","file_url":"https://github.com/shicaiwei123/mmanet/blob/HEAD/classification/lib/model_develop_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cabd997ca3e75a0c"}},{"code_sha256_prefix":"6eff21bf5e138d84","entry":"calc_accuracy_kd_patch_feature","repo":"shicaiwei123/mmanet","repo_kind":"official","path":"classification/lib/model_develop.py","file_url":"https://github.com/shicaiwei123/mmanet/blob/HEAD/classification/lib/model_develop.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6eff21bf5e138d84"}},{"code_sha256_prefix":"7aaa500285ea7919","entry":"calc_accuracy_multi","repo":"shicaiwei123/mmanet","repo_kind":"official","path":"classification/lib/model_develop.py","file_url":"https://github.com/shicaiwei123/mmanet/blob/HEAD/classification/lib/model_develop.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7aaa500285ea7919"}},{"code_sha256_prefix":"c25e8291013b4d7a","entry":"deploy_base","repo":"shicaiwei123/mmanet","repo_kind":"official","path":"classification/lib/model_develop_utils.py","file_url":"https://github.com/shicaiwei123/mmanet/blob/HEAD/classification/lib/model_develop_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c25e8291013b4d7a"}},{"code_sha256_prefix":"c761b677ad5bff9f","entry":"get_dataself_hist","repo":"shicaiwei123/mmanet","repo_kind":"official","path":"classification/lib/processing_utils.py","file_url":"https://github.com/shicaiwei123/mmanet/blob/HEAD/classification/lib/processing_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c761b677ad5bff9f"}},{"code_sha256_prefix":"1d960d0ef5c18eca","entry":"get_file_list","repo":"shicaiwei123/mmanet","repo_kind":"official","path":"classification/lib/processing_utils.py","file_url":"https://github.com/shicaiwei123/mmanet/blob/HEAD/classification/lib/processing_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1d960d0ef5c18eca"}},{"code_sha256_prefix":"25d573d05669c93c","entry":"get_mean_std","repo":"shicaiwei123/mmanet","repo_kind":"official","path":"classification/lib/processing_utils.py","file_url":"https://github.com/shicaiwei123/mmanet/blob/HEAD/classification/lib/processing_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"25d573d05669c93c"}},{"code_sha256_prefix":"5f64bc843defdf1d","entry":"get_optimizer","repo":"shicaiwei123/mmanet","repo_kind":"official","path":"segmentation/train_missing_mmanet.py","file_url":"https://github.com/shicaiwei123/mmanet/blob/HEAD/segmentation/train_missing_mmanet.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5f64bc843defdf1d"}},{"code_sha256_prefix":"fe28c9769df8105c","entry":"modality_drop","repo":"shicaiwei123/mmanet","repo_kind":"official","path":"classification/lib/model_arch.py","file_url":"https://github.com/shicaiwei123/mmanet/blob/HEAD/classification/lib/model_arch.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fe28c9769df8105c"}},{"code_sha256_prefix":"3e8fb118d35e458d","entry":"plot_embedding","repo":"shicaiwei123/mmanet","repo_kind":"official","path":"classification/lib/model_develop_utils.py","file_url":"https://github.com/shicaiwei123/mmanet/blob/HEAD/classification/lib/model_develop_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3e8fb118d35e458d"}},{"code_sha256_prefix":"26659e123368870c","entry":"preprocess_training_image","repo":"shicaiwei123/mmanet","repo_kind":"official","path":"segmentation/imagenet_pretraining.py","file_url":"https://github.com/shicaiwei123/mmanet/blob/HEAD/segmentation/imagenet_pretraining.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"26659e123368870c"}},{"code_sha256_prefix":"c6367223384ebf8a","entry":"preprocess_validation_image","repo":"shicaiwei123/mmanet","repo_kind":"official","path":"segmentation/imagenet_pretraining.py","file_url":"https://github.com/shicaiwei123/mmanet/blob/HEAD/segmentation/imagenet_pretraining.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c6367223384ebf8a"}},{"code_sha256_prefix":"4cb7014dd1d9596d","entry":"unbalance_modality_drop","repo":"shicaiwei123/mmanet","repo_kind":"official","path":"classification/lib/model_arch.py","file_url":"https://github.com/shicaiwei123/mmanet/blob/HEAD/classification/lib/model_arch.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4cb7014dd1d9596d"}},{"code_sha256_prefix":"991117cf598c2b50","entry":"validate","repo":"shicaiwei123/mmanet","repo_kind":"official","path":"segmentation/train_missing_mmanet.py","file_url":"https://github.com/shicaiwei123/mmanet/blob/HEAD/segmentation/train_missing_mmanet.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"991117cf598c2b50"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}