{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/vid-tldr-training-free-token-merging-for","title":"vid-TLDR: Training Free Token merging for Light-weight Video Transformer","arxiv_id":"2403.13347","date":"2024-03-20","proceeding":"CVPR 2024 1","authors":["Joonmyung Choi","Sanghyeok Lee","Jaewon Chu","Minhyuk Choi","Hyunwoo J. Kim"],"abstract":"Video Transformers have become the prevalent solution for various video downstream tasks with superior expressive power and flexibility. However, these video transformers suffer from heavy computational costs induced by the massive number of tokens across the entire video frames, which has been the major barrier to training the model. Further, the patches irrelevant to the main contents, e.g., backgrounds, degrade the generalization performance of models. To tackle these issues, we propose training free token merging for lightweight video Transformer (vid-TLDR) that aims to enhance the efficiency of video Transformers by merging the background tokens without additional training. For vid-TLDR, we introduce a novel approach to capture the salient regions in videos only with the attention map. Further, we introduce the saliency-aware token merging strategy by dropping the background tokens and sharpening the object scores. Our experiments show that vid-TLDR significantly mitigates the computational complexity of video Transformers while achieving competitive performance compared to the base model without vid-TLDR. Code is available at https://github.com/mlvlab/vid-TLDR.","url_abs":"https://arxiv.org/abs/2403.13347v2","url_pdf":"https://arxiv.org/pdf/2403.13347v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"vid-tldr-training-free-token-merging-for","repo_url":"https://github.com/mlvlab/vid-tldr","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"action-recognition-in-videos","task_name":"Action Recognition"},{"task_slug":"computational-efficiency","task_name":"Computational Efficiency"},{"task_slug":"text-retrieval","task_name":"Text Retrieval"},{"task_slug":"video-question-answering","task_name":"Video Question Answering"},{"task_slug":"video-retrieval","task_name":"Video Retrieval"},{"task_slug":"video-text-retrieval","task_name":"Video-Text Retrieval"},{"task_slug":"visual-question-answering","task_name":"Visual Question Answering (VQA)"},{"task_slug":"zeroshot-video-question-answer","task_name":"Zero-Shot Video Question Answer"},{"task_slug":"zero-shot-video-retrieval","task_name":"Zero-Shot Video Retrieval"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/video-retrieval-on-activitynet","task":"Video Retrieval","dataset":"ActivityNet","model":"vid-TLDR (UMT-L)","rank_in_archive_order":7,"of":31,"metrics":{"text-to-video R@1":"66.7","text-to-video R@10":"94.4","text-to-video R@5":"88.6","video-to-text R@1":"63.9","video-to-text R@10":"94.5","video-to-text R@5":"88.7"},"uses_additional_data":true},{"leaderboard":"/sota/video-retrieval-on-didemo","task":"Video Retrieval","dataset":"DiDeMo","model":"vid-TLDR (UMT-L)","rank_in_archive_order":2,"of":40,"metrics":{"text-to-video R@1":"72.3","text-to-video R@10":"94.2","text-to-video R@5":"91.2","video-to-text R@1":"68.5","video-to-text R@10":"93.8","video-to-text R@5":"89.8"},"uses_additional_data":true},{"leaderboard":"/sota/video-retrieval-on-lsmdc","task":"Video Retrieval","dataset":"LSMDC","model":"vid-TLDR (UMT-L)","rank_in_archive_order":2,"of":38,"metrics":{"text-to-video R@1":"43.1","text-to-video R@10":"71.4","text-to-video R@5":"64.5","video-to-text R@1":"40.7","video-to-text R@10":"63.6","video-to-text R@5":"70.2"},"uses_additional_data":true},{"leaderboard":"/sota/video-retrieval-on-msr-vtt","task":"Video Retrieval","dataset":"MSR-VTT","model":"vid-TLDR (UMT-L)","rank_in_archive_order":6,"of":40,"metrics":{"text-to-video R@1":"58.1","text-to-video R@10":"81.6","text-to-video R@5":"81.0","video-to-text R@1":"58.7","video-to-text R@10":"86.9","video-to-text R@5":"81.6"},"uses_additional_data":true},{"leaderboard":"/sota/video-retrieval-on-msvd","task":"Video Retrieval","dataset":"MSVD","model":"vid-TLDR (UMT-L)","rank_in_archive_order":5,"of":24,"metrics":{"text-to-video R@1":"57.9","text-to-video R@10":"89.4","text-to-video R@5":"83.8","video-to-text R@1":"82.7","video-to-text R@10":"96.3","video-to-text R@5":"94.5"},"uses_additional_data":true},{"leaderboard":"/sota/video-retrieval-on-ssv2-label-retrieval","task":"Video Retrieval","dataset":"SSv2-label retrieval","model":"vid-TLDR (UMT-L)","rank_in_archive_order":2,"of":5,"metrics":{"text-to-video R@1":"73.1","text-to-video R@10":"96.6","text-to-video R@5":"93.3"},"uses_additional_data":true},{"leaderboard":"/sota/video-retrieval-on-ssv2-template-retrieval","task":"Video Retrieval","dataset":"SSv2-template retrieval","model":"vid-TLDR (UMT-L)","rank_in_archive_order":2,"of":5,"metrics":{"text-to-video R@1":"90.2","text-to-video R@10":"100.0","text-to-video R@5":"100.0"},"uses_additional_data":true},{"leaderboard":"/sota/visual-question-answering-on-msrvtt-qa-1","task":"Visual Question Answering (VQA)","dataset":"MSRVTT-QA","model":"vid-TLDR (UMT-L)","rank_in_archive_order":9,"of":34,"metrics":{"Accuracy":"0.470"},"uses_additional_data":true},{"leaderboard":"/sota/visual-question-answering-on-msvd-qa-1","task":"Visual Question Answering (VQA)","dataset":"MSVD-QA","model":"vid-TLDR (UMT-L)","rank_in_archive_order":14,"of":36,"metrics":{"Accuracy":"0.549"},"uses_additional_data":true},{"leaderboard":"/sota/zero-shot-video-retrieval-on-activitynet","task":"Zero-Shot Video Retrieval","dataset":"ActivityNet","model":"vid-TLDR (UMT-L)","rank_in_archive_order":5,"of":12,"metrics":{"text-to-video R@1":"42.8","text-to-video R@10":"79.6","text-to-video R@5":"69.4","video-to-text R@1":"41.2","video-to-text R@10":"79.1","video-to-text R@5":"68.2"},"uses_additional_data":true},{"leaderboard":"/sota/zero-shot-video-retrieval-on-didemo","task":"Zero-Shot Video Retrieval","dataset":"DiDeMo","model":"vid-TLDR (UMT-L)","rank_in_archive_order":5,"of":26,"metrics":{"text-to-video R@1":"52.0","text-to-video R@10":"81.0","text-to-video R@5":"74.0","video-to-text R@1":"52.0","video-to-text R@10":"83.8","video-to-text R@5":"75.9"},"uses_additional_data":true},{"leaderboard":"/sota/zero-shot-video-retrieval-on-msr-vtt","task":"Zero-Shot Video Retrieval","dataset":"MSR-VTT","model":"vid-TLDR (UMT-L)","rank_in_archive_order":12,"of":41,"metrics":{"text-to-video R@1":"42.1","text-to-video R@10":"72.4","text-to-video R@5":"63.9","video-to-text R@1":"37.7","video-to-text R@10":"69.4","video-to-text R@5":"59.8"},"uses_additional_data":true},{"leaderboard":"/sota/zero-shot-video-retrieval-on-msvd","task":"Zero-Shot Video Retrieval","dataset":"MSVD","model":"vid-TLDR (UMT-L)","rank_in_archive_order":6,"of":14,"metrics":{"text-to-video R@1":"50.0","text-to-video R@10":"85.5","text-to-video R@5":"77.6","video-to-text R@1":"75.7","video-to-text R@10":"95.1","video-to-text R@5":"90.0"},"uses_additional_data":true}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2403.13347","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.13347"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mlvlab/vid-tldr","reach":null}],"summary":{"ran_draft_wrong":2,"ran_fixture":1,"unverified":1},"by_repo_kind":{"official":{"samples":4,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"23e1c4ddde84286f","entry":"get_objective_score","repo":"mlvlab/vid-tldr","repo_kind":"official","path":"multi_modality/utils/vid_TLDR.py","file_url":"https://github.com/mlvlab/vid-tldr/blob/HEAD/multi_modality/utils/vid_TLDR.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"23e1c4ddde84286f"}},{"code_sha256_prefix":"35ee8782a6d238a8","entry":"merging","repo":"mlvlab/vid-tldr","repo_kind":"official","path":"multi_modality/utils/vid_TLDR.py","file_url":"https://github.com/mlvlab/vid-tldr/blob/HEAD/multi_modality/utils/vid_TLDR.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"35ee8782a6d238a8"}},{"code_sha256_prefix":"d9d992feb83108d2","entry":"vidTLDR","repo":"mlvlab/vid-tldr","repo_kind":"official","path":"multi_modality/utils/vid_TLDR.py","file_url":"https://github.com/mlvlab/vid-tldr/blob/HEAD/multi_modality/utils/vid_TLDR.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d9d992feb83108d2"}},{"code_sha256_prefix":"5c2feb7328e8ea26","entry":"merge_wavg","repo":"mlvlab/vid-tldr","repo_kind":"official","path":"multi_modality/utils/vid_TLDR.py","file_url":"https://github.com/mlvlab/vid-tldr/blob/HEAD/multi_modality/utils/vid_TLDR.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5c2feb7328e8ea26"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}