{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/llava-onevision-easy-visual-task-transfer","title":"LLaVA-OneVision: Easy Visual Task Transfer","arxiv_id":"2408.03326","date":"2024-08-06","proceeding":null,"authors":["Bo Li","Yuanhan Zhang","Dong Guo","Renrui Zhang","Feng Li","Hao Zhang","Kaichen Zhang","Peiyuan Zhang","Yanwei Li","Ziwei Liu","Chunyuan Li"],"abstract":"We present LLaVA-OneVision, a family of open large multimodal models (LMMs) developed by consolidating our insights into data, models, and visual representations in the LLaVA-NeXT blog series. Our experimental results demonstrate that LLaVA-OneVision is the first single model that can simultaneously push the performance boundaries of open LMMs in three important computer vision scenarios: single-image, multi-image, and video scenarios. Importantly, the design of LLaVA-OneVision allows strong transfer learning across different modalities/scenarios, yielding new emerging capabilities. In particular, strong video understanding and cross-scenario capabilities are demonstrated through task transfer from images to videos.","url_abs":"https://arxiv.org/abs/2408.03326v3","url_pdf":"https://arxiv.org/pdf/2408.03326v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"llava-onevision-easy-visual-task-transfer","repo_url":"https://github.com/evolvinglmms-lab/lmms-eval","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"llava-onevision-easy-visual-task-transfer","repo_url":"https://github.com/MindSpore-scientific-2/code-14/tree/main/llava_next","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null}],"tasks":[{"task_slug":"3d-question-answering-3d-qa","task_name":"3D Question Answering (3D-QA)"},{"task_slug":"multiple-choice","task_name":"Multiple-choice"},{"task_slug":"temporal-relation-extraction","task_name":"Temporal Relation Extraction"},{"task_slug":"transfer-learning","task_name":"Transfer Learning"},{"task_slug":"video-question-answering","task_name":"Video Question Answering"},{"task_slug":"video-understanding","task_name":"Video Understanding"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"},{"task_slug":"visual-question-answering","task_name":"Visual Question Answering (VQA)"},{"task_slug":"zeroshot-video-question-answer","task_name":"Zero-Shot Video Question Answer"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/on-implicitqa","task":"","dataset":"ImplicitQA","model":"LLaVA-OneVision - 7B","rank_in_archive_order":4,"of":7,"metrics":{"Average Accuracy":"43.4","Macro Average Accuracy":"46.4"},"uses_additional_data":false},{"leaderboard":"/sota/3d-question-answering-3d-qa-on-sqa3d","task":"3D Question Answering (3D-QA)","dataset":"SQA3D","model":"LLaVA-NeXT-Video","rank_in_archive_order":13,"of":13,"metrics":{"Exact Match":"34.2"},"uses_additional_data":false},{"leaderboard":"/sota/3d-question-answering-3d-qa-on-scanqa-test-w","task":"3D Question Answering (3D-QA)","dataset":"ScanQA Test w/ objects","model":"LLaVA-NeXT-Video","rank_in_archive_order":16,"of":18,"metrics":{"BLEU-4":"9.8","CIDEr":"46.2","Exact Match":"18.7","METEOR":"9.1","ROUGE":"27.8"},"uses_additional_data":false},{"leaderboard":"/sota/temporal-relation-extraction-on-vinoground","task":"Temporal Relation Extraction","dataset":"Vinoground","model":"LLaVA-OneVision-Qwen2-72B","rank_in_archive_order":4,"of":24,"metrics":{"Group Score":"21.8","Text Score":"48.4","Video Score":"35.2"},"uses_additional_data":false},{"leaderboard":"/sota/temporal-relation-extraction-on-vinoground","task":"Temporal Relation Extraction","dataset":"Vinoground","model":"LLaVA-OneVision-Qwen2-7B","rank_in_archive_order":5,"of":24,"metrics":{"Group Score":"14.6","Text Score":"41.6","Video Score":"29.4"},"uses_additional_data":false},{"leaderboard":"/sota/video-question-answering-on-next-qa","task":"Video Question Answering","dataset":"NExT-QA","model":"LLaVA-OV(72B)","rank_in_archive_order":13,"of":47,"metrics":{"Accuracy":"80.2"},"uses_additional_data":false},{"leaderboard":"/sota/video-question-answering-on-next-qa","task":"Video Question Answering","dataset":"NExT-QA","model":"LLaVA-OV(7B)","rank_in_archive_order":15,"of":47,"metrics":{"Accuracy":"79.4"},"uses_additional_data":false},{"leaderboard":"/sota/video-question-answering-on-ovbench","task":"Video Question Answering","dataset":"OVBench","model":"LLaVA-OneVision (7B)","rank_in_archive_order":5,"of":16,"metrics":{"AVG":"49.5"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-mm-vet","task":"Visual Question Answering","dataset":"MM-Vet","model":"LLaVA-OneVision-72B","rank_in_archive_order":26,"of":231,"metrics":{"GPT-4 score":"63.7"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-mm-vet","task":"Visual Question Answering","dataset":"MM-Vet","model":"LLaVA-OneVision-7B","rank_in_archive_order":43,"of":231,"metrics":{"GPT-4 score":"57.5"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-mm-vet","task":"Visual Question Answering","dataset":"MM-Vet","model":"LLaVA-OneVision-0.5B","rank_in_archive_order":212,"of":231,"metrics":{"GPT-4 score":"29.1"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-v-bench","task":"Visual Question Answering","dataset":"V*bench","model":"LLaVA-OneVision7B","rank_in_archive_order":5,"of":5,"metrics":{"Accuracy":"74.46"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-vqa-on-vlm2-bench","task":"Visual Question Answering (VQA)","dataset":"VLM2-Bench","model":"LLaVA-OneVision-7B","rank_in_archive_order":8,"of":9,"metrics":{"Average Score on VLM2-bench (9 subtasks)":"39.35","GC-mat":"16.60","GC-trk":"13.70","OC-cnt":"56.17","OC-cpr":"47.22","OC-grp":"27.50","PC-VID":"47.25","PC-cnt":"46.67","PC-cpr":"62.00","PC-grp":"37.00"},"uses_additional_data":false},{"leaderboard":"/sota/zero-shot-video-question-answer-on-vnbench","task":"Zero-Shot Video Question Answer","dataset":"VNBench","model":"LLaVA-OneVision-72B","rank_in_archive_order":3,"of":9,"metrics":{"Accuracy":"58.7"},"uses_additional_data":false},{"leaderboard":"/sota/zero-shot-video-question-answer-on-vnbench","task":"Zero-Shot Video Question Answer","dataset":"VNBench","model":"LLaVA-OneVision-7B","rank_in_archive_order":4,"of":9,"metrics":{"Accuracy":"51.8"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2408.03326","mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}