{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/vila-u-a-unified-foundation-model-integrating","title":"VILA-U: a Unified Foundation Model Integrating Visual Understanding and Generation","arxiv_id":"2409.04429","date":"2024-09-06","proceeding":null,"authors":["Yecheng Wu","Zhuoyang Zhang","Junyu Chen","Haotian Tang","Dacheng Li","Yunhao Fang","Ligeng Zhu","Enze Xie","Hongxu Yin","Li Yi","Song Han","Yao Lu"],"abstract":"VILA-U is a Unified foundation model that integrates Video, Image, Language understanding and generation. Traditional visual language models (VLMs) use separate modules for understanding and generating visual content, which can lead to misalignment and increased complexity. In contrast, VILA-U employs a single autoregressive next-token prediction framework for both tasks, eliminating the need for additional components like diffusion models. This approach not only simplifies the model but also achieves near state-of-the-art performance in visual language understanding and generation. The success of VILA-U is attributed to two main factors: the unified vision tower that aligns discrete visual tokens with textual inputs during pretraining, which enhances visual perception, and autoregressive image generation can achieve similar quality as diffusion models with high-quality dataset. This allows VILA-U to perform comparably to more complex models using a fully token-based autoregressive framework.","url_abs":"https://arxiv.org/abs/2409.04429v2","url_pdf":"https://arxiv.org/pdf/2409.04429v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"vila-u-a-unified-foundation-model-integrating","repo_url":"https://github.com/mit-han-lab/vila-u","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"image-generation","task_name":"Image Generation"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2409.04429","atlas_url":"https://app.syntology.ai/?focus=2409.04429","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2409.04429"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mit-han-lab/vila-u","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":5,"unverified":2},"by_repo_kind":{"official":{"samples":7,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"7edecde48cb876d8","entry":"build_llm_and_tokenizer","repo":"mit-han-lab/vila-u","repo_kind":"official","path":"vila_u/model/language_model/builder.py","file_url":"https://github.com/mit-han-lab/vila-u/blob/HEAD/vila_u/model/language_model/builder.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7edecde48cb876d8"}},{"code_sha256_prefix":"2e470faeeb1e0979","entry":"context_length_extension","repo":"mit-han-lab/vila-u","repo_kind":"official","path":"vila_u/model/language_model/builder.py","file_url":"https://github.com/mit-han-lab/vila-u/blob/HEAD/vila_u/model/language_model/builder.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2e470faeeb1e0979"}},{"code_sha256_prefix":"94815e9616152485","entry":"get_frame_from_vcap","repo":"mit-han-lab/vila-u","repo_kind":"official","path":"vila_u/mm_utils.py","file_url":"https://github.com/mit-han-lab/vila-u/blob/HEAD/vila_u/mm_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"94815e9616152485"}},{"code_sha256_prefix":"11b7b07963f16286","entry":"get_model_config","repo":"mit-han-lab/vila-u","repo_kind":"official","path":"vila_u/model/utils.py","file_url":"https://github.com/mit-han-lab/vila-u/blob/HEAD/vila_u/model/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"11b7b07963f16286"}},{"code_sha256_prefix":"c1efca00f5418683","entry":"opencv_extract_frames","repo":"mit-han-lab/vila-u","repo_kind":"official","path":"vila_u/mm_utils.py","file_url":"https://github.com/mit-han-lab/vila-u/blob/HEAD/vila_u/mm_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c1efca00f5418683"}},{"code_sha256_prefix":"6ef3dc934bf9933a","entry":"build_vision_tower","repo":"mit-han-lab/vila-u","repo_kind":"official","path":"vila_u/model/multimodal_encoder/builder.py","file_url":"https://github.com/mit-han-lab/vila-u/blob/HEAD/vila_u/model/multimodal_encoder/builder.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6ef3dc934bf9933a"}},{"code_sha256_prefix":"0dde2e782959c0bd","entry":"load_image_from_base64","repo":"mit-han-lab/vila-u","repo_kind":"official","path":"vila_u/mm_utils.py","file_url":"https://github.com/mit-han-lab/vila-u/blob/HEAD/vila_u/mm_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0dde2e782959c0bd"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}