{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/boosting-multimodal-large-language-models","title":"Boosting Multimodal Large Language Models with Visual Tokens Withdrawal for Rapid Inference","arxiv_id":"2405.05803","date":"2024-05-09","proceeding":null,"authors":["Zhihang Lin","Mingbao Lin","Luxi Lin","Rongrong Ji"],"abstract":"Multimodal large language models (MLLMs) demand considerable computations for inference due to the extensive parameters and the additional input tokens needed for visual information representation. Herein, we introduce Visual Tokens Withdrawal (VTW), a plug-and-play module to boost MLLMs for rapid inference. Our approach is inspired by two intriguing phenomena we have observed: (1) the attention sink phenomenon that is prevalent in LLMs also persists in MLLMs, suggesting that initial tokens and nearest tokens receive the majority of attention, while middle vision tokens garner minimal attention in deep layers; (2) the presence of information migration, which implies that visual information is transferred to subsequent text tokens within the first few layers of MLLMs. As per our findings, we conclude that vision tokens are unnecessary in the deep layers of MLLMs. Thus, we strategically withdraw them at a certain layer, enabling only text tokens to engage in subsequent layers. To pinpoint the ideal layer for VTW, we initially analyze a limited set of tiny datasets and choose the first layer that meets the Kullback-Leibler divergence criterion. Our VTW approach can cut computational overhead by over 40\\% across diverse multimodal tasks while maintaining performance.","url_abs":"https://arxiv.org/abs/2405.05803v3","url_pdf":"https://arxiv.org/pdf/2405.05803v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"boosting-multimodal-large-language-models","repo_url":"https://github.com/lzhxmu/vtw","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[],"methods":[{"method_slug":"set","method_name":"SET"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2405.05803","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.05803"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/lzhxmu/vtw","reach":{"status":"ok"}}],"summary":{"ran_fixture":6,"ran":2,"ran_draft_wrong":1},"by_repo_kind":{"official":{"samples":9,"ran":9,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":9,"samples":[{"code_sha256_prefix":"30d7eec482ebf6b1","entry":"repeat_kv","repo":"lzhxmu/vtw","repo_kind":"official","path":"LISA-VTW/my_modeling_llama.py","file_url":"https://github.com/lzhxmu/vtw/blob/HEAD/LISA-VTW/my_modeling_llama.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":2,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"30d7eec482ebf6b1"}},{"code_sha256_prefix":"f725bc2d76076485","entry":"apply_rotary_pos_emb","repo":"lzhxmu/vtw","repo_kind":"official","path":"LISA-VTW/my_modeling_llama.py","file_url":"https://github.com/lzhxmu/vtw/blob/HEAD/LISA-VTW/my_modeling_llama.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f725bc2d76076485"}},{"code_sha256_prefix":"a1c2083ec2eaeb22","entry":"dice_loss","repo":"lzhxmu/vtw","repo_kind":"official","path":"LISA-VTW/LISA.py","file_url":"https://github.com/lzhxmu/vtw/blob/HEAD/LISA-VTW/LISA.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a1c2083ec2eaeb22"}},{"code_sha256_prefix":"7e03b180fa317c9a","entry":"divide_to_patches","repo":"lzhxmu/vtw","repo_kind":"official","path":"llava/mm_utils.py","file_url":"https://github.com/lzhxmu/vtw/blob/HEAD/llava/mm_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7e03b180fa317c9a"}},{"code_sha256_prefix":"468eedeba67f1b00","entry":"resize_and_pad_image","repo":"lzhxmu/vtw","repo_kind":"official","path":"llava/mm_utils.py","file_url":"https://github.com/lzhxmu/vtw/blob/HEAD/llava/mm_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"468eedeba67f1b00"}},{"code_sha256_prefix":"b99eea6376d1e212","entry":"rotate_half","repo":"lzhxmu/vtw","repo_kind":"official","path":"LISA-VTW/my_modeling_llama.py","file_url":"https://github.com/lzhxmu/vtw/blob/HEAD/LISA-VTW/my_modeling_llama.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b99eea6376d1e212"}},{"code_sha256_prefix":"3999ff487573f32c","entry":"select_best_resolution","repo":"lzhxmu/vtw","repo_kind":"official","path":"llava/mm_utils.py","file_url":"https://github.com/lzhxmu/vtw/blob/HEAD/llava/mm_utils.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3999ff487573f32c"}},{"code_sha256_prefix":"a9292f5d89194794","entry":"sigmoid_ce_loss","repo":"lzhxmu/vtw","repo_kind":"official","path":"LISA-VTW/LISA.py","file_url":"https://github.com/lzhxmu/vtw/blob/HEAD/LISA-VTW/LISA.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a9292f5d89194794"}},{"code_sha256_prefix":"55c32993da87759b","entry":"unpad_image","repo":"lzhxmu/vtw","repo_kind":"official","path":"llava/model/llava_arch.py","file_url":"https://github.com/lzhxmu/vtw/blob/HEAD/llava/model/llava_arch.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"55c32993da87759b"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}