{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/llava-ph-efficient-multi-modal-assistant-with","title":"LLaVA-Phi: Efficient Multi-Modal Assistant with Small Language Model","arxiv_id":"2401.02330","date":"2024-01-04","proceeding":null,"authors":["Yichen Zhu","Minjie Zhu","Ning Liu","Zhicai Ou","Xiaofeng Mou","Jian Tang"],"abstract":"In this paper, we introduce LLaVA-$\\phi$ (LLaVA-Phi), an efficient multi-modal assistant that harnesses the power of the recently advanced small language model, Phi-2, to facilitate multi-modal dialogues. LLaVA-Phi marks a notable advancement in the realm of compact multi-modal models. It demonstrates that even smaller language models, with as few as 2.7B parameters, can effectively engage in intricate dialogues that integrate both textual and visual elements, provided they are trained with high-quality corpora. Our model delivers commendable performance on publicly available benchmarks that encompass visual comprehension, reasoning, and knowledge-based perception. Beyond its remarkable performance in multi-modal dialogue tasks, our model opens new avenues for applications in time-sensitive environments and systems that require real-time interaction, such as embodied agents. It highlights the potential of smaller language models to achieve sophisticated levels of understanding and interaction, while maintaining greater resource efficiency.The project is available at {https://github.com/zhuyiche/llava-phi}.","url_abs":"https://arxiv.org/abs/2401.02330v4","url_pdf":"https://arxiv.org/pdf/2401.02330v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"llava-ph-efficient-multi-modal-assistant-with","repo_url":"https://github.com/zhuyiche/llava-phi","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"small-language-model","task_name":"Small Language Model"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/visual-question-answering-on-mm-vet","task":"Visual Question Answering","dataset":"MM-Vet","model":"LLaVA-Phi","rank_in_archive_order":214,"of":231,"metrics":{"GPT-4 score":"28.9"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2401.02330","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.02330"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/zhuyiche/llava-phi","reach":{"status":"ok"}}],"summary":{"ran_draft_wrong":3,"ran_violates":1,"ran":1,"unverified":4},"by_repo_kind":{"official":{"samples":9,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":9,"samples":[{"code_sha256_prefix":"592b3c1a88f93d7c","entry":"expand2square","repo":"zhuyiche/llava-phi","repo_kind":"official","path":"llava_phi/mm_utils.py","file_url":"https://github.com/zhuyiche/llava-phi/blob/HEAD/llava_phi/mm_utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"592b3c1a88f93d7c"}},{"code_sha256_prefix":"42a46570620cd9fa","entry":"get_chunk","repo":"zhuyiche/llava-phi","repo_kind":"official","path":"llava_phi/eval/model_vqa.py","file_url":"https://github.com/zhuyiche/llava-phi/blob/HEAD/llava_phi/eval/model_vqa.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"42a46570620cd9fa"}},{"code_sha256_prefix":"bae18947b56f2be1","entry":"is_none","repo":"zhuyiche/llava-phi","repo_kind":"official","path":"llava_phi/eval/model_vqa_mmbench.py","file_url":"https://github.com/zhuyiche/llava-phi/blob/HEAD/llava_phi/eval/model_vqa_mmbench.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bae18947b56f2be1"}},{"code_sha256_prefix":"c3ee9d07c900dd55","entry":"load_image_from_base64","repo":"zhuyiche/llava-phi","repo_kind":"official","path":"llava_phi/mm_utils.py","file_url":"https://github.com/zhuyiche/llava-phi/blob/HEAD/llava_phi/mm_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c3ee9d07c900dd55"}},{"code_sha256_prefix":"076c252c52cbb161","entry":"split_list","repo":"zhuyiche/llava-phi","repo_kind":"official","path":"llava_phi/eval/model_vqa.py","file_url":"https://github.com/zhuyiche/llava-phi/blob/HEAD/llava_phi/eval/model_vqa.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"076c252c52cbb161"}},{"code_sha256_prefix":"fa1225dfac92bc0d","entry":"get_peft_state_maybe_zero_3","repo":"zhuyiche/llava-phi","repo_kind":"official","path":"llava_phi/train/convert_model2base_llava_phi.py","file_url":"https://github.com/zhuyiche/llava-phi/blob/HEAD/llava_phi/train/convert_model2base_llava_phi.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fa1225dfac92bc0d"}},{"code_sha256_prefix":"1c53657305b66e9f","entry":"get_peft_state_non_lora_maybe_zero_3","repo":"zhuyiche/llava-phi","repo_kind":"official","path":"llava_phi/train/convert_model2base_llava_phi.py","file_url":"https://github.com/zhuyiche/llava-phi/blob/HEAD/llava_phi/train/convert_model2base_llava_phi.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1c53657305b66e9f"}},{"code_sha256_prefix":"616ffbdc154ed2d8","entry":"maybe_zero_3","repo":"zhuyiche/llava-phi","repo_kind":"official","path":"llava_phi/train/convert_model2base_llava_phi.py","file_url":"https://github.com/zhuyiche/llava-phi/blob/HEAD/llava_phi/train/convert_model2base_llava_phi.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"616ffbdc154ed2d8"}},{"code_sha256_prefix":"344dff4791fd1381","entry":"process_images","repo":"zhuyiche/llava-phi","repo_kind":"official","path":"llava_phi/mm_utils.py","file_url":"https://github.com/zhuyiche/llava-phi/blob/HEAD/llava_phi/mm_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"344dff4791fd1381"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}