{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/vtimellm-empower-llm-to-grasp-video-moments","title":"VTimeLLM: Empower LLM to Grasp Video Moments","arxiv_id":"2311.18445","date":"2023-11-30","proceeding":"CVPR 2024 1","authors":["Bin Huang","Xin Wang","Hong Chen","Zihan Song","Wenwu Zhu"],"abstract":"Large language models (LLMs) have shown remarkable text understanding capabilities, which have been extended as Video LLMs to handle video data for comprehending visual details. However, existing Video LLMs can only provide a coarse description of the entire video, failing to capture the precise start and end time boundary of specific events. In this paper, we solve this issue via proposing VTimeLLM, a novel Video LLM designed for fine-grained video moment understanding and reasoning with respect to time boundary. Specifically, our VTimeLLM adopts a boundary-aware three-stage training strategy, which respectively utilizes image-text pairs for feature alignment, multiple-event videos to increase temporal-boundary awareness, and high-quality video-instruction tuning to further improve temporal understanding ability as well as align with human intents. Extensive experiments demonstrate that in fine-grained time-related comprehension tasks for videos such as Temporal Video Grounding and Dense Video Captioning, VTimeLLM significantly outperforms existing Video LLMs. Besides, benefits from the fine-grained temporal understanding of the videos further enable VTimeLLM to beat existing Video LLMs in video dialogue benchmark, showing its superior cross-modal understanding and reasoning abilities.","url_abs":"https://arxiv.org/abs/2311.18445v1","url_pdf":"https://arxiv.org/pdf/2311.18445v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"vtimellm-empower-llm-to-grasp-video-moments","repo_url":"https://github.com/huangb23/vtimellm","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}}],"tasks":[{"task_slug":"dense-video-captioning","task_name":"Dense Video Captioning"},{"task_slug":"temporal-relation-extraction","task_name":"Temporal Relation Extraction"},{"task_slug":"vcgbench-diverse","task_name":"VCGBench-Diverse"},{"task_slug":"video-captioning","task_name":"Video Captioning"},{"task_slug":"video-grounding","task_name":"Video Grounding"},{"task_slug":"video-question-answering","task_name":"Video Question Answering"},{"task_slug":"video-based-generative-performance","task_name":"Video-based Generative Performance Benchmarking"},{"task_slug":"video-based-generative-performance-5","task_name":"Video-based Generative Performance Benchmarking (Consistency)"},{"task_slug":"video-based-generative-performance-3","task_name":"Video-based Generative Performance Benchmarking (Contextual Understanding)"},{"task_slug":"video-based-generative-performance-1","task_name":"Video-based Generative Performance Benchmarking (Correctness of Information)"},{"task_slug":"video-based-generative-performance-2","task_name":"Video-based Generative Performance Benchmarking (Detail Orientation))"},{"task_slug":"video-based-generative-performance-4","task_name":"Video-based Generative Performance Benchmarking (Temporal Understanding)"}],"methods":[{"method_slug":"align","method_name":"ALIGN"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/dense-video-captioning-on-activitynet","task":"Dense Video Captioning","dataset":"ActivityNet Captions","model":"VTimeLLM","rank_in_archive_order":12,"of":12,"metrics":{"CIDEr":"27.6","SODA":"5.8"},"uses_additional_data":false},{"leaderboard":"/sota/temporal-relation-extraction-on-vinoground","task":"Temporal Relation Extraction","dataset":"Vinoground","model":"VTimeLLM","rank_in_archive_order":21,"of":24,"metrics":{"Group Score":"5.2","Text Score":"19.4","Video Score":"27"},"uses_additional_data":false},{"leaderboard":"/sota/vcgbench-diverse-on-videoinstruct","task":"VCGBench-Diverse","dataset":"VideoInstruct","model":"VTimeLLM","rank_in_archive_order":5,"of":6,"metrics":{"Consistency":"2.35","Contextual Understanding":"2.48","Correctness of Information":"2.16","Dense Captioning":"1.13","Detail Orientation":"2.41","Reasoning":"3.45","Spatial Understanding":"2.29","Temporal Understanding":"1.46","mean":"2.17"},"uses_additional_data":false},{"leaderboard":"/sota/video-question-answering-on-ovbench","task":"Video Question Answering","dataset":"OVBench","model":"VTimeLLM (7B)","rank_in_archive_order":11,"of":16,"metrics":{"AVG":"33.1"},"uses_additional_data":false},{"leaderboard":"/sota/video-based-generative-performance","task":"Video-based Generative Performance Benchmarking","dataset":"VideoInstruct","model":"VTimeLLM","rank_in_archive_order":17,"of":23,"metrics":{"Consistency":"2.47","Contextual Understanding":"3.40","Correctness of Information":"2.78","Detail Orientation":"3.10","Temporal Understanding":"2.49","mean":"2.85"},"uses_additional_data":false},{"leaderboard":"/sota/video-based-generative-performance-2","task":"Video-based Generative Performance Benchmarking (Consistency)","dataset":"VideoInstruct","model":"VTimeLLM","rank_in_archive_order":11,"of":18,"metrics":{"gpt-score":"2.47"},"uses_additional_data":false},{"leaderboard":"/sota/video-based-generative-performance-3","task":"Video-based Generative Performance Benchmarking (Contextual Understanding)","dataset":"VideoInstruct","model":"VTimeLLM","rank_in_archive_order":11,"of":18,"metrics":{"gpt-score":"3.40"},"uses_additional_data":false},{"leaderboard":"/sota/video-based-generative-performance-1","task":"Video-based Generative Performance Benchmarking (Correctness of Information)","dataset":"VideoInstruct","model":"VTimeLLM","rank_in_archive_order":11,"of":18,"metrics":{"gpt-score":"2.78"},"uses_additional_data":false},{"leaderboard":"/sota/video-based-generative-performance-4","task":"Video-based Generative Performance Benchmarking (Detail Orientation))","dataset":"VideoInstruct","model":"VTimeLLM","rank_in_archive_order":4,"of":18,"metrics":{"gpt-score":"3.10"},"uses_additional_data":false},{"leaderboard":"/sota/video-based-generative-performance-5","task":"Video-based Generative Performance Benchmarking (Temporal Understanding)","dataset":"VideoInstruct","model":"VTimeLLM","rank_in_archive_order":10,"of":18,"metrics":{"gpt-score":"2.49"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2311.18445","atlas_url":"https://app.syntology.ai/?focus=2311.18445","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.18445"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/huangb23/vtimellm","reach":{"status":"ok","spdx":"NOASSERTION"}}],"summary":{"ran_fixture":1,"ran":3,"ran_draft_wrong":1,"unverified":6},"by_repo_kind":{"official":{"samples":11,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":11,"samples":[{"code_sha256_prefix":"5cfeaee89a75c5ff","entry":"apply_rotary_pos_emb","repo":"huangb23/vtimellm","repo_kind":"official","path":"vtimellm/model/chatglm/modeling_chatglm.py","file_url":"https://github.com/huangb23/vtimellm/blob/HEAD/vtimellm/model/chatglm/modeling_chatglm.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5cfeaee89a75c5ff"}},{"code_sha256_prefix":"f6eec537f45dfb96","entry":"grounding_metrics","repo":"huangb23/vtimellm","repo_kind":"official","path":"vtimellm/eval/metric.py","file_url":"https://github.com/huangb23/vtimellm/blob/HEAD/vtimellm/eval/metric.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"f6eec537f45dfb96"}},{"code_sha256_prefix":"c3ee9d07c900dd55","entry":"load_image_from_base64","repo":"huangb23/vtimellm","repo_kind":"official","path":"vtimellm/mm_utils.py","file_url":"https://github.com/huangb23/vtimellm/blob/HEAD/vtimellm/mm_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c3ee9d07c900dd55"}},{"code_sha256_prefix":"1df990c375318896","entry":"process_images","repo":"huangb23/vtimellm","repo_kind":"official","path":"vtimellm/mm_utils.py","file_url":"https://github.com/huangb23/vtimellm/blob/HEAD/vtimellm/mm_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1df990c375318896"}},{"code_sha256_prefix":"d7e378ee7899fbde","entry":"split_tensor_along_last_dim","repo":"huangb23/vtimellm","repo_kind":"official","path":"vtimellm/model/chatglm/modeling_chatglm.py","file_url":"https://github.com/huangb23/vtimellm/blob/HEAD/vtimellm/model/chatglm/modeling_chatglm.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d7e378ee7899fbde"}},{"code_sha256_prefix":"237019e36b9481d9","entry":"default_init","repo":"huangb23/vtimellm","repo_kind":"official","path":"vtimellm/model/chatglm/modeling_chatglm.py","file_url":"https://github.com/huangb23/vtimellm/blob/HEAD/vtimellm/model/chatglm/modeling_chatglm.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"237019e36b9481d9"}},{"code_sha256_prefix":"d6b600df5a8550bc","entry":"load_lora","repo":"huangb23/vtimellm","repo_kind":"official","path":"vtimellm/model/builder.py","file_url":"https://github.com/huangb23/vtimellm/blob/HEAD/vtimellm/model/builder.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d6b600df5a8550bc"}},{"code_sha256_prefix":"3d00e5b35fde051d","entry":"load_pretrained_model","repo":"huangb23/vtimellm","repo_kind":"official","path":"vtimellm/model/builder.py","file_url":"https://github.com/huangb23/vtimellm/blob/HEAD/vtimellm/model/builder.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"3d00e5b35fde051d"}},{"code_sha256_prefix":"362a9306f09e0d58","entry":"merge_similar_sentences","repo":"huangb23/vtimellm","repo_kind":"official","path":"vtimellm/eval/metric.py","file_url":"https://github.com/huangb23/vtimellm/blob/HEAD/vtimellm/eval/metric.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"362a9306f09e0d58"}},{"code_sha256_prefix":"37899f22fb191b37","entry":"pretty_print_semaphore","repo":"huangb23/vtimellm","repo_kind":"official","path":"vtimellm/utils.py","file_url":"https://github.com/huangb23/vtimellm/blob/HEAD/vtimellm/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"37899f22fb191b37"}},{"code_sha256_prefix":"f9939a84b9a65279","entry":"violates_moderation","repo":"huangb23/vtimellm","repo_kind":"official","path":"vtimellm/utils.py","file_url":"https://github.com/huangb23/vtimellm/blob/HEAD/vtimellm/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f9939a84b9a65279"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}