{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/llama-vid-an-image-is-worth-2-tokens-in-large","title":"LLaMA-VID: An Image is Worth 2 Tokens in Large Language Models","arxiv_id":"2311.17043","date":"2023-11-28","proceeding":null,"authors":["Yanwei Li","Chengyao Wang","Jiaya Jia"],"abstract":"In this work, we present a novel method to tackle the token generation challenge in Vision Language Models (VLMs) for video and image understanding, called LLaMA-VID. Current VLMs, while proficient in tasks like image captioning and visual question answering, face computational burdens when processing long videos due to the excessive visual tokens. LLaMA-VID addresses this issue by representing each frame with two distinct tokens, namely context token and content token. The context token encodes the overall image context based on user input, whereas the content token encapsulates visual cues in each frame. This dual-token strategy significantly reduces the overload of long videos while preserving critical information. Generally, LLaMA-VID empowers existing frameworks to support hour-long videos and pushes their upper limit with an extra context token. It is proved to surpass previous methods on most of video- or image-based benchmarks. Code is available https://github.com/dvlab-research/LLaMA-VID}{https://github.com/dvlab-research/LLaMA-VID","url_abs":"https://arxiv.org/abs/2311.17043v1","url_pdf":"https://arxiv.org/pdf/2311.17043v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"llama-vid-an-image-is-worth-2-tokens-in-large","repo_url":"https://github.com/dvlab-research/llama-vid","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"llama-vid-an-image-is-worth-2-tokens-in-large","repo_url":"https://github.com/lastmile-ai/aiconfig","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"none","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"image-captioning","task_name":"Image Captioning"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"video-question-answering","task_name":"Video Question Answering"},{"task_slug":"video-based-generative-performance","task_name":"Video-based Generative Performance Benchmarking"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"},{"task_slug":"zeroshot-video-question-answer","task_name":"Zero-Shot Video Question Answer"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/video-question-answering-on-activitynet-qa","task":"Video Question Answering","dataset":"ActivityNet-QA","model":"LLaMA-VID-13B (2 Token)","rank_in_archive_order":11,"of":36,"metrics":{"Accuracy":"47.5","Confidence score":"3.3"},"uses_additional_data":false},{"leaderboard":"/sota/video-question-answering-on-activitynet-qa","task":"Video Question Answering","dataset":"ActivityNet-QA","model":"LLaMA-VID-7B (2 Token)","rank_in_archive_order":12,"of":36,"metrics":{"Accuracy":"47.4","Confidence score":"3.3"},"uses_additional_data":false},{"leaderboard":"/sota/video-question-answering-on-ovbench","task":"Video Question Answering","dataset":"OVBench","model":"LLaMA-VID (7B)","rank_in_archive_order":9,"of":16,"metrics":{"AVG":"41.9"},"uses_additional_data":false},{"leaderboard":"/sota/video-based-generative-performance","task":"Video-based Generative Performance Benchmarking","dataset":"VideoInstruct","model":"LLaMA-VID-13B (2 Token)","rank_in_archive_order":13,"of":23,"metrics":{"Consistency":"2.63","Contextual Understanding":"3.60","Correctness of Information":"3.07","Detail Orientation":"3.05","Temporal Understanding":"2.58","mean":"2.99"},"uses_additional_data":false},{"leaderboard":"/sota/video-based-generative-performance","task":"Video-based Generative Performance Benchmarking","dataset":"VideoInstruct","model":"LLaMA-VID-7B (2 Token)","rank_in_archive_order":16,"of":23,"metrics":{"Consistency":"2.51","Contextual Understanding":"3.53","Correctness of Information":"2.96","Detail Orientation":"3.00","Temporal Understanding":"2.46","mean":"2.89"},"uses_additional_data":false},{"leaderboard":"/sota/zeroshot-video-question-answer-on-activitynet","task":"Zero-Shot Video Question Answer","dataset":"ActivityNet-QA","model":"LLaMA-VID-13B (2 Token)","rank_in_archive_order":15,"of":28,"metrics":{"Accuracy":"47.5","Confidence Score":"3.3"},"uses_additional_data":false},{"leaderboard":"/sota/zeroshot-video-question-answer-on-activitynet","task":"Zero-Shot Video Question Answer","dataset":"ActivityNet-QA","model":"LLaMA-VID-7B (2 Token)","rank_in_archive_order":16,"of":28,"metrics":{"Accuracy":"47.4","Confidence Score":"3.3"},"uses_additional_data":false},{"leaderboard":"/sota/zeroshot-video-question-answer-on-msrvtt-qa","task":"Zero-Shot Video Question Answer","dataset":"MSRVTT-QA","model":"LLaMA-VID-13B (2 Token)","rank_in_archive_order":18,"of":30,"metrics":{"Accuracy":"58.9","Confidence Score":"3.3"},"uses_additional_data":false},{"leaderboard":"/sota/zeroshot-video-question-answer-on-msrvtt-qa","task":"Zero-Shot Video Question Answer","dataset":"MSRVTT-QA","model":"LLaMA-VID-7B (2 Token)","rank_in_archive_order":19,"of":30,"metrics":{"Accuracy":"57.7","Confidence Score":"3.2"},"uses_additional_data":false},{"leaderboard":"/sota/zeroshot-video-question-answer-on-msvd-qa","task":"Zero-Shot Video Question Answer","dataset":"MSVD-QA","model":"LLaMA-VID-13B (2 Token)","rank_in_archive_order":19,"of":28,"metrics":{"Accuracy":"70.0","Confidence Score":"3.7"},"uses_additional_data":false},{"leaderboard":"/sota/zeroshot-video-question-answer-on-msvd-qa","task":"Zero-Shot Video Question Answer","dataset":"MSVD-QA","model":"LLaMA-VID-7B (2 Token)","rank_in_archive_order":20,"of":28,"metrics":{"Accuracy":"69.7","Confidence Score":"3.7"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2311.17043","atlas_url":"https://app.syntology.ai/?focus=2311.17043","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.17043"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/dvlab-research/llama-vid","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/lastmile-ai/aiconfig","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":2,"ran_violates":1,"unverified":1},"by_repo_kind":{"official":{"samples":4,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"42a46570620cd9fa","entry":"get_chunk","repo":"dvlab-research/llama-vid","repo_kind":"official","path":"llamavid/eval/model_activitynet_qa.py","file_url":"https://github.com/dvlab-research/llama-vid/blob/HEAD/llamavid/eval/model_activitynet_qa.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"42a46570620cd9fa"}},{"code_sha256_prefix":"bae18947b56f2be1","entry":"is_none","repo":"dvlab-research/llama-vid","repo_kind":"official","path":"llamavid/eval/model_vqa_mmbench.py","file_url":"https://github.com/dvlab-research/llama-vid/blob/HEAD/llamavid/eval/model_vqa_mmbench.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bae18947b56f2be1"}},{"code_sha256_prefix":"076c252c52cbb161","entry":"split_list","repo":"dvlab-research/llama-vid","repo_kind":"official","path":"llamavid/eval/model_activitynet_qa.py","file_url":"https://github.com/dvlab-research/llama-vid/blob/HEAD/llamavid/eval/model_activitynet_qa.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"076c252c52cbb161"}},{"code_sha256_prefix":"d367874245639411","entry":"apply_rotary_pos_emb_inference","repo":"dvlab-research/llama-vid","repo_kind":"official","path":"llamavid/train/llama_flash_attn_monkey_patch.py","file_url":"https://github.com/dvlab-research/llama-vid/blob/HEAD/llamavid/train/llama_flash_attn_monkey_patch.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d367874245639411"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}