{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/onellm-one-framework-to-align-all-modalities","title":"OneLLM: One Framework to Align All Modalities with Language","arxiv_id":"2312.03700","date":"2023-12-06","proceeding":"CVPR 2024 1","authors":["Jiaming Han","Kaixiong Gong","Yiyuan Zhang","Jiaqi Wang","Kaipeng Zhang","Dahua Lin","Yu Qiao","Peng Gao","Xiangyu Yue"],"abstract":"Multimodal large language models (MLLMs) have gained significant attention due to their strong multimodal understanding capability. However, existing works rely heavily on modality-specific encoders, which usually differ in architecture and are limited to common modalities. In this paper, we present OneLLM, an MLLM that aligns eight modalities to language using a unified framework. We achieve this through a unified multimodal encoder and a progressive multimodal alignment pipeline. In detail, we first train an image projection module to connect a vision encoder with LLM. Then, we build a universal projection module (UPM) by mixing multiple image projection modules and dynamic routing. Finally, we progressively align more modalities to LLM with the UPM. To fully leverage the potential of OneLLM in following instructions, we also curated a comprehensive multimodal instruction dataset, including 2M items from image, audio, video, point cloud, depth/normal map, IMU and fMRI brain activity. OneLLM is evaluated on 25 diverse benchmarks, encompassing tasks such as multimodal captioning, question answering and reasoning, where it delivers excellent performance. Code, data, model and online demo are available at https://github.com/csuhan/OneLLM","url_abs":"https://arxiv.org/abs/2312.03700v2","url_pdf":"https://arxiv.org/pdf/2312.03700v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"onellm-one-framework-to-align-all-modalities","repo_url":"https://github.com/csuhan/onellm","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}}],"tasks":[{"task_slug":"all","task_name":"All"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"}],"methods":[{"method_slug":"align","method_name":"ALIGN"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/visual-question-answering-on-mm-vet","task":"Visual Question Answering","dataset":"MM-Vet","model":"OneLLM-7B","rank_in_archive_order":211,"of":231,"metrics":{"GPT-4 score":"29.1","Params":"7B"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2312.03700","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.03700"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/csuhan/onellm","reach":{"status":"ok","spdx":"NOASSERTION"}}],"summary":{"ran_fixture":2,"ran":1,"ran_violates":1},"by_repo_kind":{"official":{"samples":4,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":4,"samples":[{"code_sha256_prefix":"b47d48e431b34acd","entry":"apply_rotary_emb","repo":"csuhan/onellm","repo_kind":"official","path":"model/LLM/onellm.py","file_url":"https://github.com/csuhan/onellm/blob/HEAD/model/LLM/onellm.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b47d48e431b34acd"}},{"code_sha256_prefix":"423344b082006dfe","entry":"load_fmri","repo":"csuhan/onellm","repo_kind":"official","path":"eval/fmri_cap_nsd.py","file_url":"https://github.com/csuhan/onellm/blob/HEAD/eval/fmri_cap_nsd.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"423344b082006dfe"}},{"code_sha256_prefix":"14a84c2cbfebc413","entry":"precompute_freqs_cis","repo":"csuhan/onellm","repo_kind":"official","path":"model/LLM/onellm.py","file_url":"https://github.com/csuhan/onellm/blob/HEAD/model/LLM/onellm.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"14a84c2cbfebc413"}},{"code_sha256_prefix":"70bf6ebaafd266c4","entry":"reshape_for_broadcast","repo":"csuhan/onellm","repo_kind":"official","path":"model/LLM/onellm.py","file_url":"https://github.com/csuhan/onellm/blob/HEAD/model/LLM/onellm.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"70bf6ebaafd266c4"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}