{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/healnet-hybrid-multi-modal-fusion-for","title":"HEALNet: Multimodal Fusion for Heterogeneous Biomedical Data","arxiv_id":"2311.09115","date":"2023-11-15","proceeding":null,"authors":["Konstantin Hemker","Nikola Simidjievski","Mateja Jamnik"],"abstract":"Technological advances in medical data collection, such as high-throughput genomic sequencing and digital high-resolution histopathology, have contributed to the rising requirement for multimodal biomedical modelling, specifically for image, tabular and graph data. Most multimodal deep learning approaches use modality-specific architectures that are often trained separately and cannot capture the crucial cross-modal information that motivates the integration of different data sources. This paper presents the Hybrid Early-fusion Attention Learning Network (HEALNet): a flexible multimodal fusion architecture, which a) preserves modality-specific structural information, b) captures the cross-modal interactions and structural information in a shared latent space, c) can effectively handle missing modalities during training and inference, and d) enables intuitive model inspection by learning on the raw data input instead of opaque embeddings. We conduct multimodal survival analysis on Whole Slide Images and Multi-omic data on four cancer datasets from The Cancer Genome Atlas (TCGA). HEALNet achieves state-of-the-art performance compared to other end-to-end trained fusion models, substantially improving over unimodal and multimodal baselines whilst being robust in scenarios with missing modalities.","url_abs":"https://arxiv.org/abs/2311.09115v3","url_pdf":"https://arxiv.org/pdf/2311.09115v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"healnet-hybrid-multi-modal-fusion-for","repo_url":"https://github.com/konst-int-i/healnet","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"healnet-hybrid-multi-modal-fusion-for","repo_url":"https://github.com/konst-int-i/mm-lego","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"multimodal-deep-learning","task_name":"Multimodal Deep Learning"},{"task_slug":"survival-analysis","task_name":"Survival Analysis"},{"task_slug":"whole-slide-images","task_name":"whole slide images"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2311.09115","atlas_url":"https://app.syntology.ai/?focus=2311.09115","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.09115"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/konst-int-i/healnet","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/konst-int-i/mm-lego","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":2,"ran_draft_wrong":2,"ran_honours":1,"ran_violates":3,"unverified":3},"by_repo_kind":{"official":{"samples":3,"ran":3,"repositories":1},"listed":{"samples":8,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"21c59012b6a28663","entry":"Reg_Block","repo":"konst-int-i/mm-lego","repo_kind":"listed","path":"mm_lego/models/baselines/mcat.py","file_url":"https://github.com/konst-int-i/mm-lego/blob/HEAD/mm_lego/models/baselines/mcat.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"21c59012b6a28663"}},{"code_sha256_prefix":"10482673211edd0c","entry":"SNN_Block","repo":"konst-int-i/mm-lego","repo_kind":"listed","path":"mm_lego/models/baselines/mcat.py","file_url":"https://github.com/konst-int-i/mm-lego/blob/HEAD/mm_lego/models/baselines/mcat.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"10482673211edd0c"}},{"code_sha256_prefix":"b64b66cc00d70b2f","entry":"cache_fn","repo":"konst-int-i/healnet","repo_kind":"official","path":"healnet/models/healnet.py","file_url":"https://github.com/konst-int-i/healnet/blob/HEAD/healnet/models/healnet.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b64b66cc00d70b2f"}},{"code_sha256_prefix":"13404bd487501696","entry":"default","repo":"konst-int-i/healnet","repo_kind":"official","path":"healnet/models/healnet.py","file_url":"https://github.com/konst-int-i/healnet/blob/HEAD/healnet/models/healnet.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"13404bd487501696"}},{"code_sha256_prefix":"60fff7c3c400d7ff","entry":"default","repo":"konst-int-i/mm-lego","repo_kind":"listed","path":"mm_lego/models/model_utils.py","file_url":"https://github.com/konst-int-i/mm-lego/blob/HEAD/mm_lego/models/model_utils.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"60fff7c3c400d7ff"}},{"code_sha256_prefix":"5b44cb09cc8a80a0","entry":"exists","repo":"konst-int-i/healnet","repo_kind":"official","path":"healnet/models/healnet.py","file_url":"https://github.com/konst-int-i/healnet/blob/HEAD/healnet/models/healnet.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5b44cb09cc8a80a0"}},{"code_sha256_prefix":"aa5486a3650902d8","entry":"exists","repo":"konst-int-i/mm-lego","repo_kind":"listed","path":"mm_lego/models/model_utils.py","file_url":"https://github.com/konst-int-i/mm-lego/blob/HEAD/mm_lego/models/model_utils.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"aa5486a3650902d8"}},{"code_sha256_prefix":"97c161d1309960d6","entry":"l1_loss","repo":"konst-int-i/mm-lego","repo_kind":"listed","path":"mm_lego/models/losses.py","file_url":"https://github.com/konst-int-i/mm-lego/blob/HEAD/mm_lego/models/losses.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"97c161d1309960d6"}},{"code_sha256_prefix":"e56401f8aa027f99","entry":"cache_fn","repo":"konst-int-i/mm-lego","repo_kind":"listed","path":"mm_lego/models/model_utils.py","file_url":"https://github.com/konst-int-i/mm-lego/blob/HEAD/mm_lego/models/model_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e56401f8aa027f99"}},{"code_sha256_prefix":"78f0d5d3d3b8e58a","entry":"get_model_parameters","repo":"konst-int-i/mm-lego","repo_kind":"listed","path":"mm_lego/pipeline/base.py","file_url":"https://github.com/konst-int-i/mm-lego/blob/HEAD/mm_lego/pipeline/base.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"78f0d5d3d3b8e58a"}},{"code_sha256_prefix":"8109ef4e167f0ad9","entry":"nll_loss","repo":"konst-int-i/mm-lego","repo_kind":"listed","path":"mm_lego/models/losses.py","file_url":"https://github.com/konst-int-i/mm-lego/blob/HEAD/mm_lego/models/losses.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8109ef4e167f0ad9"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}