{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/mplug-2-a-modularized-multi-modal-foundation","title":"mPLUG-2: A Modularized Multi-modal Foundation Model Across Text, Image and Video","arxiv_id":"2302.00402","date":"2023-02-01","proceeding":null,"authors":["Haiyang Xu","Qinghao Ye","Ming Yan","Yaya Shi","Jiabo Ye","Yuanhong Xu","Chenliang Li","Bin Bi","Qi Qian","Wei Wang","Guohai Xu","Ji Zhang","Songfang Huang","Fei Huang","Jingren Zhou"],"abstract":"Recent years have witnessed a big convergence of language, vision, and multi-modal pretraining. In this work, we present mPLUG-2, a new unified paradigm with modularized design for multi-modal pretraining, which can benefit from modality collaboration while addressing the problem of modality entanglement. In contrast to predominant paradigms of solely relying on sequence-to-sequence generation or encoder-based instance discrimination, mPLUG-2 introduces a multi-module composition network by sharing common universal modules for modality collaboration and disentangling different modality modules to deal with modality entanglement. It is flexible to select different modules for different understanding and generation tasks across all modalities including text, image, and video. Empirical study shows that mPLUG-2 achieves state-of-the-art or competitive results on a broad range of over 30 downstream tasks, spanning multi-modal tasks of image-text and video-text understanding and generation, and uni-modal tasks of text-only, image-only, and video-only understanding. Notably, mPLUG-2 shows new state-of-the-art results of 48.0 top-1 accuracy and 80.3 CIDEr on the challenging MSRVTT video QA and video caption tasks with a far smaller model size and data scale. It also demonstrates strong zero-shot transferability on vision-language and video-language tasks. Code and models will be released in https://github.com/alibaba/AliceMind.","url_abs":"https://arxiv.org/abs/2302.00402v1","url_pdf":"https://arxiv.org/pdf/2302.00402v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"mplug-2-a-modularized-multi-modal-foundation","repo_url":"https://github.com/alibaba/AliceMind","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":null},{"paper_slug":"mplug-2-a-modularized-multi-modal-foundation","repo_url":"https://github.com/X-PLUG/mPLUG-2","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"mplug-2-a-modularized-multi-modal-foundation","repo_url":"https://github.com/x-plug/mplug-owl","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"mplug-2-a-modularized-multi-modal-foundation","repo_url":"https://github.com/modelscope/modelscope","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"action-classification","task_name":"Action Classification"},{"task_slug":"image-classification","task_name":"Image Classification"},{"task_slug":"image-retrieval","task_name":"Image Retrieval"},{"task_slug":"tgif-frame","task_name":"TGIF-Frame"},{"task_slug":"video-captioning","task_name":"Video Captioning"},{"task_slug":"video-question-answering","task_name":"Video Question Answering"},{"task_slug":"video-retrieval","task_name":"Video Retrieval"},{"task_slug":"visual-grounding","task_name":"Visual Grounding"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"},{"task_slug":"visual-question-answering","task_name":"Visual Question Answering (VQA)"},{"task_slug":"zero-shot-video-retrieval","task_name":"Zero-Shot Video Retrieval"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/action-classification-on-kinetics-400","task":"Action Classification","dataset":"Kinetics-400","model":"mPLUG-2","rank_in_archive_order":41,"of":207,"metrics":{"Acc@1":"87.1","Acc@5":"97.7"},"uses_additional_data":false},{"leaderboard":"/sota/action-classification-on-kinetics-600","task":"Action Classification","dataset":"Kinetics-600","model":"mPLUG-2","rank_in_archive_order":12,"of":65,"metrics":{"Top-1 Accuracy":"89.8","Top-5 Accuracy":"98.3"},"uses_additional_data":true},{"leaderboard":"/sota/action-classification-on-kinetics-700","task":"Action Classification","dataset":"Kinetics-700","model":"mPLUG-2","rank_in_archive_order":13,"of":36,"metrics":{"Top-1 Accuracy":"80.4","Top-5 Accuracy":"94.9"},"uses_additional_data":true},{"leaderboard":"/sota/video-captioning-on-msr-vtt-1","task":"Video Captioning","dataset":"MSR-VTT","model":"mPLUG-2","rank_in_archive_order":1,"of":24,"metrics":{"BLEU-4":"57.8","CIDEr":"80.0","METEOR":"34.9","ROUGE-L":"70.1"},"uses_additional_data":false},{"leaderboard":"/sota/video-captioning-on-msvd-1","task":"Video Captioning","dataset":"MSVD","model":"mPLUG-2","rank_in_archive_order":5,"of":16,"metrics":{"BLEU-4":"70.5","CIDEr":"165.8","METEOR":"48.4","ROUGE-L":"85.3"},"uses_additional_data":false},{"leaderboard":"/sota/video-question-answering-on-msrvtt-qa","task":"Video Question Answering","dataset":"MSRVTT-QA","model":"mPLUG-2","rank_in_archive_order":6,"of":14,"metrics":{"Accuracy":"48.0"},"uses_additional_data":false},{"leaderboard":"/sota/video-retrieval-on-didemo","task":"Video Retrieval","dataset":"DiDeMo","model":"mPLUG-2","rank_in_archive_order":15,"of":40,"metrics":{"text-to-video R@1":"56.4","text-to-video R@10":"85.2","text-to-video R@5":"79.1"},"uses_additional_data":true},{"leaderboard":"/sota/video-retrieval-on-lsmdc","task":"Video Retrieval","dataset":"LSMDC","model":"mPLUG-2","rank_in_archive_order":6,"of":38,"metrics":{"text-to-video R@1":"34.4","text-to-video R@10":"65.1","text-to-video R@5":"55.2"},"uses_additional_data":true},{"leaderboard":"/sota/video-retrieval-on-msr-vtt-1ka","task":"Video Retrieval","dataset":"MSR-VTT-1kA","model":"mPLUG-2","rank_in_archive_order":12,"of":63,"metrics":{"text-to-video R@1":"53.1","text-to-video R@10":"84.7","text-to-video R@5":"77.6"},"uses_additional_data":false},{"leaderboard":"/sota/visual-grounding-on-refcoco-test-b","task":"Visual Grounding","dataset":"RefCOCO+ test B","model":"mPLUG-2","rank_in_archive_order":2,"of":6,"metrics":{"Accuracy (%)":"86.05"},"uses_additional_data":false},{"leaderboard":"/sota/visual-grounding-on-refcoco-testa","task":"Visual Grounding","dataset":"RefCOCO+ testA","model":"mPLUG-2","rank_in_archive_order":2,"of":7,"metrics":{"Accuracy (%)":"92.8"},"uses_additional_data":false},{"leaderboard":"/sota/visual-grounding-on-refcoco-val","task":"Visual Grounding","dataset":"RefCOCO+ val","model":"mPLUG-2","rank_in_archive_order":2,"of":6,"metrics":{"Accuracy (%)":"90.33"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-vqa-v2-test-dev-1","task":"Visual Question Answering","dataset":"VQA v2 test-dev","model":"mPLUG-2","rank_in_archive_order":6,"of":11,"metrics":{"Accuracy":"81.11"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-msrvtt-qa-1","task":"Visual Question Answering (VQA)","dataset":"MSRVTT-QA","model":"mPLUG-2","rank_in_archive_order":3,"of":34,"metrics":{"Accuracy":"0.480"},"uses_additional_data":true},{"leaderboard":"/sota/visual-question-answering-on-msvd-qa-1","task":"Visual Question Answering (VQA)","dataset":"MSVD-QA","model":"mPLUG-2","rank_in_archive_order":7,"of":36,"metrics":{"Accuracy":"0.581"},"uses_additional_data":true},{"leaderboard":"/sota/zero-shot-video-retrieval-on-didemo","task":"Zero-Shot Video Retrieval","dataset":"DiDeMo","model":"mPLUG-2","rank_in_archive_order":7,"of":26,"metrics":{"text-to-video R@1":"45.7","text-to-video R@10":"79.2","text-to-video R@5":"71.1"},"uses_additional_data":false},{"leaderboard":"/sota/zero-shot-video-retrieval-on-lsmdc","task":"Zero-Shot Video Retrieval","dataset":"LSMDC","model":"mPLUG-2","rank_in_archive_order":5,"of":16,"metrics":{"text-to-video R@1":"24.1","text-to-video R@10":"52.0","text-to-video R@5":"43.8"},"uses_additional_data":false},{"leaderboard":"/sota/zero-shot-video-retrieval-on-msr-vtt","task":"Zero-Shot Video Retrieval","dataset":"MSR-VTT","model":"mPLUG-2","rank_in_archive_order":7,"of":41,"metrics":{"text-to-video R@1":"47.1","text-to-video R@10":"79.0","text-to-video R@5":"69.7"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2302.00402","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2302.00402"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/modelscope/modelscope","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/X-PLUG/mPLUG-2","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/alibaba/AliceMind","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/x-plug/mplug-owl","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":5,"ran_fixture":3,"ran":1,"unverified":10},"by_repo_kind":{"listed":{"samples":19,"ran":9,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"cf9ffa02a42184af","entry":"whitespace_tokenize","repo":"X-PLUG/mPLUG-2","repo_kind":"listed","path":"models/tokenization_bert.py","file_url":"https://github.com/X-PLUG/mPLUG-2/blob/HEAD/models/tokenization_bert.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":2,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cf9ffa02a42184af"}},{"code_sha256_prefix":"aa81d03f07735ab8","entry":"compute_acc","repo":"X-PLUG/mPLUG-2","repo_kind":"listed","path":"utils.py","file_url":"https://github.com/X-PLUG/mPLUG-2/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"aa81d03f07735ab8"}},{"code_sha256_prefix":"ad12a494674d23fb","entry":"compute_n_params","repo":"X-PLUG/mPLUG-2","repo_kind":"listed","path":"utils.py","file_url":"https://github.com/X-PLUG/mPLUG-2/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ad12a494674d23fb"}},{"code_sha256_prefix":"3ac6b7d76e8e3584","entry":"drop_path","repo":"X-PLUG/mPLUG-2","repo_kind":"listed","path":"models/clip_video.py","file_url":"https://github.com/X-PLUG/mPLUG-2/blob/HEAD/models/clip_video.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3ac6b7d76e8e3584"}},{"code_sha256_prefix":"45567e61fe09093a","entry":"get_ltor_masks_and_position_ids_from_embeddings","repo":"x-plug/mplug-owl","repo_kind":"listed","path":"mPLUG-Owl/mplug_owl/modeling_mplug_owl.py","file_url":"https://github.com/x-plug/mplug-owl/blob/HEAD/mPLUG-Owl/mplug_owl/modeling_mplug_owl.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"45567e61fe09093a"}},{"code_sha256_prefix":"265f4d55ccd60d18","entry":"get_media_indices","repo":"x-plug/mplug-owl","repo_kind":"listed","path":"mPLUG-Owl/mplug_owl/modeling_mplug_owl.py","file_url":"https://github.com/x-plug/mplug-owl/blob/HEAD/mPLUG-Owl/mplug_owl/modeling_mplug_owl.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"265f4d55ccd60d18"}},{"code_sha256_prefix":"c6ec173f19f5c34d","entry":"interpolate_pos_embed","repo":"X-PLUG/mPLUG-2","repo_kind":"listed","path":"models/vit.py","file_url":"https://github.com/X-PLUG/mPLUG-2/blob/HEAD/models/vit.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c6ec173f19f5c34d"}},{"code_sha256_prefix":"e7fbc7a74a3457c7","entry":"load_vocab","repo":"X-PLUG/mPLUG-2","repo_kind":"listed","path":"models/tokenization_bert.py","file_url":"https://github.com/X-PLUG/mPLUG-2/blob/HEAD/models/tokenization_bert.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e7fbc7a74a3457c7"}},{"code_sha256_prefix":"aa5a91226269bf75","entry":"tile","repo":"X-PLUG/mPLUG-2","repo_kind":"listed","path":"models/model_video_caption_mplug.py","file_url":"https://github.com/X-PLUG/mPLUG-2/blob/HEAD/models/model_video_caption_mplug.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"aa5a91226269bf75"}},{"code_sha256_prefix":"e496506b4b6756a4","entry":"build_predictor","repo":"X-PLUG/mPLUG-2","repo_kind":"listed","path":"models/predictor.py","file_url":"https://github.com/X-PLUG/mPLUG-2/blob/HEAD/models/predictor.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e496506b4b6756a4"}},{"code_sha256_prefix":"21aff36aa456493f","entry":"clamp_inf","repo":"X-PLUG/mPLUG-2","repo_kind":"listed","path":"models/modeling_mplug2.py","file_url":"https://github.com/X-PLUG/mPLUG-2/blob/HEAD/models/modeling_mplug2.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"21aff36aa456493f"}},{"code_sha256_prefix":"96590457ad0c455c","entry":"decode_int32","repo":"X-PLUG/mPLUG-2","repo_kind":"listed","path":"dataset/caption_dataset.py","file_url":"https://github.com/X-PLUG/mPLUG-2/blob/HEAD/dataset/caption_dataset.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"96590457ad0c455c"}},{"code_sha256_prefix":"3295906672ecb8a4","entry":"initialize_clip","repo":"X-PLUG/mPLUG-2","repo_kind":"listed","path":"models/visual_transformers.py","file_url":"https://github.com/X-PLUG/mPLUG-2/blob/HEAD/models/visual_transformers.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3295906672ecb8a4"}},{"code_sha256_prefix":"53b733b2695e4c7f","entry":"initialize_clip_video","repo":"X-PLUG/mPLUG-2","repo_kind":"listed","path":"models/visual_transformers.py","file_url":"https://github.com/X-PLUG/mPLUG-2/blob/HEAD/models/visual_transformers.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"53b733b2695e4c7f"}},{"code_sha256_prefix":"26be70dca3249c0b","entry":"load_tf_weights_in_bert","repo":"X-PLUG/mPLUG-2","repo_kind":"listed","path":"models/modeling_mplug2.py","file_url":"https://github.com/X-PLUG/mPLUG-2/blob/HEAD/models/modeling_mplug2.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"26be70dca3249c0b"}},{"code_sha256_prefix":"01b33ff330828e1e","entry":"resize_pos_embed","repo":"X-PLUG/mPLUG-2","repo_kind":"listed","path":"models/visual_transformers.py","file_url":"https://github.com/X-PLUG/mPLUG-2/blob/HEAD/models/visual_transformers.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"01b33ff330828e1e"}},{"code_sha256_prefix":"cfb25231bfbf746c","entry":"test_collect_fn","repo":"X-PLUG/mPLUG-2","repo_kind":"listed","path":"video_caption_mplug2.py","file_url":"https://github.com/X-PLUG/mPLUG-2/blob/HEAD/video_caption_mplug2.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cfb25231bfbf746c"}},{"code_sha256_prefix":"69b8b31e44791260","entry":"tile","repo":"X-PLUG/mPLUG-2","repo_kind":"listed","path":"models/predictor.py","file_url":"https://github.com/X-PLUG/mPLUG-2/blob/HEAD/models/predictor.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"69b8b31e44791260"}},{"code_sha256_prefix":"63e2cc5fb0616ee9","entry":"top_k_top_p_filtering","repo":"X-PLUG/mPLUG-2","repo_kind":"listed","path":"models/predictor.py","file_url":"https://github.com/X-PLUG/mPLUG-2/blob/HEAD/models/predictor.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"63e2cc5fb0616ee9"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}