{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/visionzip-longer-is-better-but-not-necessary","title":"VisionZip: Longer is Better but Not Necessary in Vision Language Models","arxiv_id":"2412.04467","date":"2024-12-05","proceeding":"CVPR 2025 1","authors":["Senqiao Yang","Yukang Chen","Zhuotao Tian","Chengyao Wang","Jingyao Li","Bei Yu","Jiaya Jia"],"abstract":"Recent advancements in vision-language models have enhanced performance by increasing the length of visual tokens, making them much longer than text tokens and significantly raising computational costs. However, we observe that the visual tokens generated by popular vision encoders, such as CLIP and SigLIP, contain significant redundancy. To address this, we introduce VisionZip, a simple yet effective method that selects a set of informative tokens for input to the language model, reducing visual token redundancy and improving efficiency while maintaining model performance. The proposed VisionZip can be widely applied to image and video understanding tasks and is well-suited for multi-turn dialogues in real-world scenarios, where previous methods tend to underperform. Experimental results show that VisionZip outperforms the previous state-of-the-art method by at least 5% performance gains across nearly all settings. Moreover, our method significantly enhances model inference speed, improving the prefilling time by 8x and enabling the LLaVA-Next 13B model to infer faster than the LLaVA-Next 7B model while achieving better results. Furthermore, we analyze the causes of this redundancy and encourage the community to focus on extracting better visual features rather than merely increasing token length. Our code is available at https://github.com/dvlab-research/VisionZip .","url_abs":"https://arxiv.org/abs/2412.04467v1","url_pdf":"https://arxiv.org/pdf/2412.04467v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"visionzip-longer-is-better-but-not-necessary","repo_url":"https://github.com/dvlab-research/visionzip","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"video-understanding","task_name":"Video Understanding"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"}],"methods":[{"method_slug":"clip","method_name":"CLIP"},{"method_slug":"focus","method_name":"Focus"},{"method_slug":"set","method_name":"SET"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/visual-question-answering-on-mm-vet","task":"Visual Question Answering","dataset":"MM-Vet","model":"VisionZip (Retain 128 Tokens, fine-tuning)","rank_in_archive_order":177,"of":231,"metrics":{"GPT-4 score":"32.9"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-mm-vet","task":"Visual Question Answering","dataset":"MM-Vet","model":"VisionZip (Retain 192 Tokens, fine-tuning)","rank_in_archive_order":181,"of":231,"metrics":{"GPT-4 score":"32.6"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-mm-vet","task":"Visual Question Answering","dataset":"MM-Vet","model":"VisionZip (Retain 128 Tokens)","rank_in_archive_order":182,"of":231,"metrics":{"GPT-4 score":"32.6"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-mm-vet","task":"Visual Question Answering","dataset":"MM-Vet","model":"VisionZip (Retain 192 Tokens)","rank_in_archive_order":191,"of":231,"metrics":{"GPT-4 score":"31.7"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-mm-vet","task":"Visual Question Answering","dataset":"MM-Vet","model":"VisionZip (Retain 64 Tokens)","rank_in_archive_order":192,"of":231,"metrics":{"GPT-4 score":"31.7"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-mm-vet","task":"Visual Question Answering","dataset":"MM-Vet","model":"VisionZip (Retain 64 Tokens, fine-tuning)","rank_in_archive_order":207,"of":231,"metrics":{"GPT-4 score":"30.2"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2412.04467","atlas_url":"https://app.syntology.ai/?focus=2412.04467","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2412.04467"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/dvlab-research/visionzip","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_draft_wrong":1,"unverified":10},"by_repo_kind":{"official":{"samples":11,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"e03d53ba9d4f9ae5","entry":"rotate_half","repo":"dvlab-research/visionzip","repo_kind":"official","path":"Qwen2_5_VL/qwen2_5vl_visionzip.py","file_url":"https://github.com/dvlab-research/visionzip/blob/HEAD/Qwen2_5_VL/qwen2_5vl_visionzip.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e03d53ba9d4f9ae5"}},{"code_sha256_prefix":"994ad8442c3d6ae8","entry":"CLIPAttention_forward","repo":"dvlab-research/visionzip","repo_kind":"official","path":"visionzip/utils.py","file_url":"https://github.com/dvlab-research/visionzip/blob/HEAD/visionzip/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"994ad8442c3d6ae8"}},{"code_sha256_prefix":"27a169cf3a211ef5","entry":"CLIP_EncoderLayer_forward","repo":"dvlab-research/visionzip","repo_kind":"official","path":"visionzip/utils.py","file_url":"https://github.com/dvlab-research/visionzip/blob/HEAD/visionzip/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"27a169cf3a211ef5"}},{"code_sha256_prefix":"56baaa3cae2a8e1f","entry":"apply_rotary_pos_emb_flashatt","repo":"dvlab-research/visionzip","repo_kind":"official","path":"Qwen2_5_VL/qwen2_5vl_visionzip.py","file_url":"https://github.com/dvlab-research/visionzip/blob/HEAD/Qwen2_5_VL/qwen2_5vl_visionzip.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"56baaa3cae2a8e1f"}},{"code_sha256_prefix":"50a09accb6ca2549","entry":"apply_rotary_pos_emb_vision","repo":"dvlab-research/visionzip","repo_kind":"official","path":"Qwen2_5_VL/qwen2_5vl_visionzip.py","file_url":"https://github.com/dvlab-research/visionzip/blob/HEAD/Qwen2_5_VL/qwen2_5vl_visionzip.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"50a09accb6ca2549"}},{"code_sha256_prefix":"1585cd5e445a26b9","entry":"encode_images_visionzip","repo":"dvlab-research/visionzip","repo_kind":"official","path":"visionzip/llava_arch.py","file_url":"https://github.com/dvlab-research/visionzip/blob/HEAD/visionzip/llava_arch.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1585cd5e445a26b9"}},{"code_sha256_prefix":"e8c6fd834f76c9b1","entry":"encode_images_visionzip_multi","repo":"dvlab-research/visionzip","repo_kind":"official","path":"visionzip/llava_arch.py","file_url":"https://github.com/dvlab-research/visionzip/blob/HEAD/visionzip/llava_arch.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e8c6fd834f76c9b1"}},{"code_sha256_prefix":"7db2c78b1f3cc693","entry":"load_image","repo":"dvlab-research/visionzip","repo_kind":"official","path":"gradio_demo.py","file_url":"https://github.com/dvlab-research/visionzip/blob/HEAD/gradio_demo.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7db2c78b1f3cc693"}},{"code_sha256_prefix":"a54840c2e71e73aa","entry":"parse_r","repo":"dvlab-research/visionzip","repo_kind":"official","path":"visionzip/utils.py","file_url":"https://github.com/dvlab-research/visionzip/blob/HEAD/visionzip/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a54840c2e71e73aa"}},{"code_sha256_prefix":"f170144b6e9abcec","entry":"resize_image","repo":"dvlab-research/visionzip","repo_kind":"official","path":"gradio_demo.py","file_url":"https://github.com/dvlab-research/visionzip/blob/HEAD/gradio_demo.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f170144b6e9abcec"}},{"code_sha256_prefix":"cbc15a0bfa5bc165","entry":"restore_image_features_sorted","repo":"dvlab-research/visionzip","repo_kind":"official","path":"visionzip/llava_arch.py","file_url":"https://github.com/dvlab-research/visionzip/blob/HEAD/visionzip/llava_arch.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cbc15a0bfa5bc165"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}