{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/ming-moe-enhancing-medical-multi-task","title":"MING-MOE: Enhancing Medical Multi-Task Learning in Large Language Models with Sparse Mixture of Low-Rank Adapter Experts","arxiv_id":"2404.09027","date":"2024-04-13","proceeding":null,"authors":["Yusheng Liao","Shuyang Jiang","Yu Wang","Yanfeng Wang"],"abstract":"Large language models like ChatGPT have shown substantial progress in natural language understanding and generation, proving valuable across various disciplines, including the medical field. Despite advancements, challenges persist due to the complexity and diversity inherent in medical tasks which often require multi-task learning capabilities. Previous approaches, although beneficial, fall short in real-world applications because they necessitate task-specific annotations at inference time, limiting broader generalization. This paper introduces MING-MOE, a novel Mixture-of-Expert~(MOE)-based medical large language model designed to manage diverse and complex medical tasks without requiring task-specific annotations, thus enhancing its usability across extensive datasets. MING-MOE employs a Mixture of Low-Rank Adaptation (MoLoRA) technique, allowing for efficient parameter usage by maintaining base model parameters static while adapting through a minimal set of trainable parameters. We demonstrate that MING-MOE achieves state-of-the-art (SOTA) performance on over 20 medical tasks, illustrating a significant improvement over existing models. This approach not only extends the capabilities of medical language models but also improves inference efficiency.","url_abs":"https://arxiv.org/abs/2404.09027v1","url_pdf":"https://arxiv.org/pdf/2404.09027v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"ming-moe-enhancing-medical-multi-task","repo_url":"https://github.com/mediabrain-sjtu/ming","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"ming-moe-enhancing-medical-multi-task","repo_url":"https://github.com/mediabrain-sjtu/medicalgpt-zh","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"diversity","task_name":"Diversity"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"large-language-model","task_name":"Large Language Model"},{"task_slug":"multi-task-learning","task_name":"Multi-Task Learning"},{"task_slug":"natural-language-understanding","task_name":"Natural Language Understanding"}],"methods":[{"method_slug":"base","method_name":"BASE"},{"method_slug":"set","method_name":"SET"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2404.09027","atlas_url":"https://app.syntology.ai/?focus=2404.09027","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.09027"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mediabrain-sjtu/ming","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mediabrain-sjtu/medicalgpt-zh","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":2,"ran_draft_wrong":4,"ran_honours":1,"unverified":6},"by_repo_kind":{"official":{"samples":3,"ran":3,"repositories":1},"listed":{"samples":10,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"f315af9021bb4de9","entry":"convert_to_json","repo":"mediabrain-sjtu/medicalgpt-zh","repo_kind":"listed","path":"ming/eval/eval_gpt4.py","file_url":"https://github.com/mediabrain-sjtu/medicalgpt-zh/blob/HEAD/ming/eval/eval_gpt4.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f315af9021bb4de9"}},{"code_sha256_prefix":"42a46570620cd9fa","entry":"get_chunk","repo":"mediabrain-sjtu/medicalgpt-zh","repo_kind":"listed","path":"ming/eval/model_diverse_gen.py","file_url":"https://github.com/mediabrain-sjtu/medicalgpt-zh/blob/HEAD/ming/eval/model_diverse_gen.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"42a46570620cd9fa"}},{"code_sha256_prefix":"fa6d2859d3461c9e","entry":"get_model_name_from_path","repo":"mediabrain-sjtu/medicalgpt-zh","repo_kind":"listed","path":"ming/utils.py","file_url":"https://github.com/mediabrain-sjtu/medicalgpt-zh/blob/HEAD/ming/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fa6d2859d3461c9e"}},{"code_sha256_prefix":"0133c7875f5508e2","entry":"normalize","repo":"mediabrain-sjtu/ming","repo_kind":"official","path":"ming/eval/eval_em.py","file_url":"https://github.com/mediabrain-sjtu/ming/blob/HEAD/ming/eval/eval_em.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0133c7875f5508e2"}},{"code_sha256_prefix":"ceb9b19eaecb1183","entry":"normalize_dfrac","repo":"mediabrain-sjtu/ming","repo_kind":"official","path":"ming/eval/eval_em.py","file_url":"https://github.com/mediabrain-sjtu/ming/blob/HEAD/ming/eval/eval_em.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ceb9b19eaecb1183"}},{"code_sha256_prefix":"75cb6a84ff49f59b","entry":"normalize_frac","repo":"mediabrain-sjtu/ming","repo_kind":"official","path":"ming/eval/eval_em.py","file_url":"https://github.com/mediabrain-sjtu/ming/blob/HEAD/ming/eval/eval_em.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"75cb6a84ff49f59b"}},{"code_sha256_prefix":"076c252c52cbb161","entry":"split_list","repo":"mediabrain-sjtu/medicalgpt-zh","repo_kind":"listed","path":"ming/eval/model_diverse_gen.py","file_url":"https://github.com/mediabrain-sjtu/medicalgpt-zh/blob/HEAD/ming/eval/model_diverse_gen.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"076c252c52cbb161"}},{"code_sha256_prefix":"ce4b943c1f4622e6","entry":"check_target_module_exists","repo":"mediabrain-sjtu/medicalgpt-zh","repo_kind":"listed","path":"ming/model/utils.py","file_url":"https://github.com/mediabrain-sjtu/medicalgpt-zh/blob/HEAD/ming/model/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ce4b943c1f4622e6"}},{"code_sha256_prefix":"8948016310b7f832","entry":"create_data_loader","repo":"mediabrain-sjtu/medicalgpt-zh","repo_kind":"listed","path":"ming/eval/model_diverse_gen.py","file_url":"https://github.com/mediabrain-sjtu/medicalgpt-zh/blob/HEAD/ming/eval/model_diverse_gen.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8948016310b7f832"}},{"code_sha256_prefix":"ff08a7b61f0ff773","entry":"create_mixoflora_module","repo":"mediabrain-sjtu/medicalgpt-zh","repo_kind":"listed","path":"ming/model/utils.py","file_url":"https://github.com/mediabrain-sjtu/medicalgpt-zh/blob/HEAD/ming/model/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ff08a7b61f0ff773"}},{"code_sha256_prefix":"6f3e31642958b2ee","entry":"get_default_conv_template","repo":"mediabrain-sjtu/medicalgpt-zh","repo_kind":"listed","path":"ming/conversations.py","file_url":"https://github.com/mediabrain-sjtu/medicalgpt-zh/blob/HEAD/ming/conversations.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6f3e31642958b2ee"}},{"code_sha256_prefix":"80d3d94e5c8311ac","entry":"get_mixoflora_model","repo":"mediabrain-sjtu/medicalgpt-zh","repo_kind":"listed","path":"ming/model/utils.py","file_url":"https://github.com/mediabrain-sjtu/medicalgpt-zh/blob/HEAD/ming/model/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"80d3d94e5c8311ac"}},{"code_sha256_prefix":"e9663a782ec03800","entry":"load_pretrained_model","repo":"mediabrain-sjtu/medicalgpt-zh","repo_kind":"listed","path":"ming/model/builder.py","file_url":"https://github.com/mediabrain-sjtu/medicalgpt-zh/blob/HEAD/ming/model/builder.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e9663a782ec03800"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}