{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/mome-mixture-of-multimodal-experts-for","title":"MoME: Mixture of Multimodal Experts for Generalist Multimodal Large Language Models","arxiv_id":"2407.12709","date":"2024-07-17","proceeding":null,"authors":["Leyang Shen","Gongwei Chen","Rui Shao","Weili Guan","Liqiang Nie"],"abstract":"Multimodal large language models (MLLMs) have demonstrated impressive capabilities across various vision-language tasks. However, a generalist MLLM typically underperforms compared with a specialist MLLM on most VL tasks, which can be attributed to task interference. In this paper, we propose a mixture of multimodal experts (MoME) to mitigate task interference and obtain a generalist MLLM. Our MoME is composed of two key components, a mixture of vision experts (MoVE) and a mixture of language experts (MoLE). MoVE can adaptively modulate the features transformed from various vision encoders, and has a strong compatibility in transformation architecture. MoLE incorporates sparsely gated experts into LLMs to achieve painless improvements with roughly unchanged inference costs. In response to task interference, our MoME specializes in both vision and language modality to adapt to task discrepancies. Extensive experiments show that MoME significantly improves the performance of generalist MLLMs across various VL tasks. The source code is released at https://github.com/JiuTian-VL/MoME","url_abs":"https://arxiv.org/abs/2407.12709v1","url_pdf":"https://arxiv.org/pdf/2407.12709v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"mome-mixture-of-multimodal-experts-for","repo_url":"https://github.com/jiutian-vl/mome","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2407.12709","atlas_url":"https://app.syntology.ai/?focus=2407.12709","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.12709"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jiutian-vl/mome","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":4,"ran_violates":1,"ran_draft_wrong":1,"unverified":2},"by_repo_kind":{"official":{"samples":8,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"5653e2bbfb29e58c","entry":"box_extract","repo":"jiutian-vl/mome","repo_kind":"official","path":"demo_mome.py","file_url":"https://github.com/jiutian-vl/mome/blob/HEAD/demo_mome.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5653e2bbfb29e58c"}},{"code_sha256_prefix":"4cb732f513d69dfd","entry":"disabled_train","repo":"jiutian-vl/mome","repo_kind":"official","path":"models/mome_model.py","file_url":"https://github.com/jiutian-vl/mome/blob/HEAD/models/mome_model.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4cb732f513d69dfd"}},{"code_sha256_prefix":"439482b5435bb1d9","entry":"extract_ans","repo":"jiutian-vl/mome","repo_kind":"official","path":"demo_mome.py","file_url":"https://github.com/jiutian-vl/mome/blob/HEAD/demo_mome.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"439482b5435bb1d9"}},{"code_sha256_prefix":"433e6abade4a9ee0","entry":"forward_mlp_llama","repo":"jiutian-vl/mome","repo_kind":"official","path":"models/moe_adapters/mole_adapter.py","file_url":"https://github.com/jiutian-vl/mome/blob/HEAD/models/moe_adapters/mole_adapter.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"433e6abade4a9ee0"}},{"code_sha256_prefix":"e7fcb6d9ac9deaf4","entry":"named_apply","repo":"jiutian-vl/mome","repo_kind":"official","path":"models/dinov2_models/vision_transformer.py","file_url":"https://github.com/jiutian-vl/mome/blob/HEAD/models/dinov2_models/vision_transformer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e7fcb6d9ac9deaf4"}},{"code_sha256_prefix":"eb82d58de85115b1","entry":"top1_gate","repo":"jiutian-vl/mome","repo_kind":"official","path":"models/moe_adapters/mole_adapter.py","file_url":"https://github.com/jiutian-vl/mome/blob/HEAD/models/moe_adapters/mole_adapter.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"eb82d58de85115b1"}},{"code_sha256_prefix":"6d14779e4c4e5d4f","entry":"create_eva_vit_g","repo":"jiutian-vl/mome","repo_kind":"official","path":"models/eva_vit.py","file_url":"https://github.com/jiutian-vl/mome/blob/HEAD/models/eva_vit.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6d14779e4c4e5d4f"}},{"code_sha256_prefix":"c36ac37873d36891","entry":"get_image_from_plt","repo":"jiutian-vl/mome","repo_kind":"official","path":"demo_mome.py","file_url":"https://github.com/jiutian-vl/mome/blob/HEAD/demo_mome.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c36ac37873d36891"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}