{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/a-theory-of-unimodal-bias-in-multimodal","title":"Understanding Unimodal Bias in Multimodal Deep Linear Networks","arxiv_id":"2312.00935","date":"2023-12-01","proceeding":null,"authors":["Yedi Zhang","Peter E. Latham","Andrew Saxe"],"abstract":"Using multiple input streams simultaneously to train multimodal neural networks is intuitively advantageous but practically challenging. A key challenge is unimodal bias, where a network overly relies on one modality and ignores others during joint training. We develop a theory of unimodal bias with multimodal deep linear networks to understand how architecture and data statistics influence this bias. This is the first work to calculate the duration of the unimodal phase in learning as a function of the depth at which modalities are fused within the network, dataset statistics, and initialization. We show that the deeper the layer at which fusion occurs, the longer the unimodal phase. A long unimodal phase can lead to a generalization deficit and permanent unimodal bias in the overparametrized regime. Our results, derived for multimodal linear networks, extend to nonlinear networks in certain settings. Taken together, this work illuminates pathologies of multimodal learning under joint training, showing that late and intermediate fusion architectures can give rise to long unimodal phases and permanent unimodal bias. Our code is available at: https://yedizhang.github.io/unimodal-bias.html.","url_abs":"https://arxiv.org/abs/2312.00935v2","url_pdf":"https://arxiv.org/pdf/2312.00935v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"a-theory-of-unimodal-bias-in-multimodal","repo_url":"https://github.com/yedizhang/unimodal-bias","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2312.00935","atlas_url":"https://app.syntology.ai/?focus=2312.00935","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.00935"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yedizhang/unimodal-bias","reach":{"status":"ok"}}],"summary":{"ran":11,"unverified":3},"by_repo_kind":{"official":{"samples":14,"ran":11,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":14,"samples":[{"code_sha256_prefix":"0106d7269de18775","entry":"bernoulli","repo":"yedizhang/unimodal-bias","repo_kind":"official","path":"MNIST/model.py","file_url":"https://github.com/yedizhang/unimodal-bias/blob/HEAD/MNIST/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0106d7269de18775"}},{"code_sha256_prefix":"46f4fe4903bf7dbc","entry":"gen_data","repo":"yedizhang/unimodal-bias","repo_kind":"official","path":"data.py","file_url":"https://github.com/yedizhang/unimodal-bias/blob/HEAD/data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"46f4fe4903bf7dbc"}},{"code_sha256_prefix":"fe9ec1860c82e51f","entry":"gen_toy_data","repo":"yedizhang/unimodal-bias","repo_kind":"official","path":"data.py","file_url":"https://github.com/yedizhang/unimodal-bias/blob/HEAD/data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fe9ec1860c82e51f"}},{"code_sha256_prefix":"b85f6a95e2b792fc","entry":"integrand","repo":"yedizhang/unimodal-bias","repo_kind":"official","path":"util.py","file_url":"https://github.com/yedizhang/unimodal-bias/blob/HEAD/util.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b85f6a95e2b792fc"}},{"code_sha256_prefix":"bed6665babb3e724","entry":"lag_depth","repo":"yedizhang/unimodal-bias","repo_kind":"official","path":"util.py","file_url":"https://github.com/yedizhang/unimodal-bias/blob/HEAD/util.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bed6665babb3e724"}},{"code_sha256_prefix":"57da5c99cef3dfb1","entry":"prep_axs","repo":"yedizhang/unimodal-bias","repo_kind":"official","path":"vis.py","file_url":"https://github.com/yedizhang/unimodal-bias/blob/HEAD/vis.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"57da5c99cef3dfb1"}},{"code_sha256_prefix":"c55371d1d8149f90","entry":"prep_data","repo":"yedizhang/unimodal-bias","repo_kind":"official","path":"data.py","file_url":"https://github.com/yedizhang/unimodal-bias/blob/HEAD/data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c55371d1d8149f90"}},{"code_sha256_prefix":"cf4a0be4f23f97d9","entry":"set_data","repo":"yedizhang/unimodal-bias","repo_kind":"official","path":"MNIST/model.py","file_url":"https://github.com/yedizhang/unimodal-bias/blob/HEAD/MNIST/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"cf4a0be4f23f97d9"}},{"code_sha256_prefix":"0f6a746f865b5bb5","entry":"test_unit","repo":"yedizhang/unimodal-bias","repo_kind":"official","path":"MNIST/mnist.py","file_url":"https://github.com/yedizhang/unimodal-bias/blob/HEAD/MNIST/mnist.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0f6a746f865b5bb5"}},{"code_sha256_prefix":"945265f7e1ab92b8","entry":"uni2multi","repo":"yedizhang/unimodal-bias","repo_kind":"official","path":"MNIST/model.py","file_url":"https://github.com/yedizhang/unimodal-bias/blob/HEAD/MNIST/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"945265f7e1ab92b8"}},{"code_sha256_prefix":"270370386083422b","entry":"vis_relu","repo":"yedizhang/unimodal-bias","repo_kind":"official","path":"vis.py","file_url":"https://github.com/yedizhang/unimodal-bias/blob/HEAD/vis.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"270370386083422b"}},{"code_sha256_prefix":"aaa77c921b0d4363","entry":"lag_twolayer","repo":"yedizhang/unimodal-bias","repo_kind":"official","path":"util.py","file_url":"https://github.com/yedizhang/unimodal-bias/blob/HEAD/util.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"aaa77c921b0d4363"}},{"code_sha256_prefix":"7ccb0fa52a55533f","entry":"test","repo":"yedizhang/unimodal-bias","repo_kind":"official","path":"MNIST/mnist.py","file_url":"https://github.com/yedizhang/unimodal-bias/blob/HEAD/MNIST/mnist.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7ccb0fa52a55533f"}},{"code_sha256_prefix":"ddb4f2f6ba9ecceb","entry":"train","repo":"yedizhang/unimodal-bias","repo_kind":"official","path":"MNIST/mnist.py","file_url":"https://github.com/yedizhang/unimodal-bias/blob/HEAD/MNIST/mnist.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ddb4f2f6ba9ecceb"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}