{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/beyond-text-frozen-large-language-models-in","title":"Beyond Text: Frozen Large Language Models in Visual Signal Comprehension","arxiv_id":"2403.07874","date":"2024-03-12","proceeding":"CVPR 2024 1","authors":["Lei Zhu","Fangyun Wei","Yanye Lu"],"abstract":"In this work, we investigate the potential of a large language model (LLM) to directly comprehend visual signals without the necessity of fine-tuning on multi-modal datasets. The foundational concept of our method views an image as a linguistic entity, and translates it to a set of discrete words derived from the LLM's vocabulary. To achieve this, we present the Vision-to-Language Tokenizer, abbreviated as V2T Tokenizer, which transforms an image into a ``foreign language'' with the combined aid of an encoder-decoder, the LLM vocabulary, and a CLIP model. With this innovative image encoding, the LLM gains the ability not only for visual comprehension but also for image denoising and restoration in an auto-regressive fashion-crucially, without any fine-tuning. We undertake rigorous experiments to validate our method, encompassing understanding tasks like image recognition, image captioning, and visual question answering, as well as image denoising tasks like inpainting, outpainting, deblurring, and shift restoration. Code and models are available at https://github.com/zh460045050/V2L-Tokenizer.","url_abs":"https://arxiv.org/abs/2403.07874v1","url_pdf":"https://arxiv.org/pdf/2403.07874v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"beyond-text-frozen-large-language-models-in","repo_url":"https://github.com/zh460045050/v2l-tokenizer","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"deblurring","task_name":"Deblurring"},{"task_slug":"decoder","task_name":"Decoder"},{"task_slug":"denoising","task_name":"Denoising"},{"task_slug":"image-captioning","task_name":"Image Captioning"},{"task_slug":"image-denoising","task_name":"Image Denoising"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"large-language-model","task_name":"Large Language Model"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"}],"methods":[{"method_slug":"clip","method_name":"CLIP"},{"method_slug":"set","method_name":"SET"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2403.07874","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.07874"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/zh460045050/v2l-tokenizer","reach":{"status":"ok"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/zh460045050/V2L-Tokenizer","reach":{"status":"ok"}}],"summary":{"ran":6,"ran_draft_wrong":10,"ran_honours":1,"unverified":9},"by_repo_kind":{"official":{"samples":26,"ran":17,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":26,"samples":[{"code_sha256_prefix":"83318aade307ecf5","entry":"ActNorm","repo":"zh460045050/V2L-Tokenizer","repo_kind":"official","path":"models/models_v2l.py","file_url":"https://github.com/zh460045050/V2L-Tokenizer/blob/HEAD/models/models_v2l.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"83318aade307ecf5"}},{"code_sha256_prefix":"43b458c02b16d6b9","entry":"CrossAttention","repo":"zh460045050/V2L-Tokenizer","repo_kind":"official","path":"models/models_v2l.py","file_url":"https://github.com/zh460045050/V2L-Tokenizer/blob/HEAD/models/models_v2l.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"43b458c02b16d6b9"}},{"code_sha256_prefix":"c3a6b977022957cb","entry":"Normalize","repo":"zh460045050/v2l-tokenizer","repo_kind":"official","path":"models/encoder_decoder.py","file_url":"https://github.com/zh460045050/v2l-tokenizer/blob/HEAD/models/encoder_decoder.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c3a6b977022957cb"}},{"code_sha256_prefix":"65e5a0174a2f15f6","entry":"ScalingLayer","repo":"zh460045050/V2L-Tokenizer","repo_kind":"official","path":"models/models_v2l.py","file_url":"https://github.com/zh460045050/V2L-Tokenizer/blob/HEAD/models/models_v2l.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"65e5a0174a2f15f6"}},{"code_sha256_prefix":"98f385d847636a3e","entry":"basic_clean","repo":"zh460045050/v2l-tokenizer","repo_kind":"official","path":"models/global_encoder/simple_tokenizer.py","file_url":"https://github.com/zh460045050/v2l-tokenizer/blob/HEAD/models/global_encoder/simple_tokenizer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"98f385d847636a3e"}},{"code_sha256_prefix":"a88fb62991c51ca9","entry":"get_ckpt_path","repo":"zh460045050/V2L-Tokenizer","repo_kind":"official","path":"models/models_v2l.py","file_url":"https://github.com/zh460045050/V2L-Tokenizer/blob/HEAD/models/models_v2l.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a88fb62991c51ca9"}},{"code_sha256_prefix":"e70eb36cf08faee4","entry":"get_ckpt_path","repo":"zh460045050/v2l-tokenizer","repo_kind":"official","path":"models/lpips.py","file_url":"https://github.com/zh460045050/v2l-tokenizer/blob/HEAD/models/lpips.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e70eb36cf08faee4"}},{"code_sha256_prefix":"221b2d116fdf1032","entry":"get_obj_from_str","repo":"zh460045050/v2l-tokenizer","repo_kind":"official","path":"models/models_v2l.py","file_url":"https://github.com/zh460045050/v2l-tokenizer/blob/HEAD/models/models_v2l.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"221b2d116fdf1032"}},{"code_sha256_prefix":"d919ae32e5e4e616","entry":"get_pairs","repo":"zh460045050/v2l-tokenizer","repo_kind":"official","path":"models/global_encoder/simple_tokenizer.py","file_url":"https://github.com/zh460045050/v2l-tokenizer/blob/HEAD/models/global_encoder/simple_tokenizer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d919ae32e5e4e616"}},{"code_sha256_prefix":"4b147e0a54c3fa04","entry":"load_config","repo":"zh460045050/v2l-tokenizer","repo_kind":"official","path":"step4_training_v2l_tokenizer.py","file_url":"https://github.com/zh460045050/v2l-tokenizer/blob/HEAD/step4_training_v2l_tokenizer.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4b147e0a54c3fa04"}},{"code_sha256_prefix":"07d90bd6ae1be7db","entry":"md5_hash","repo":"zh460045050/V2L-Tokenizer","repo_kind":"official","path":"models/models_v2l.py","file_url":"https://github.com/zh460045050/V2L-Tokenizer/blob/HEAD/models/models_v2l.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"07d90bd6ae1be7db"}},{"code_sha256_prefix":"3137073275f8c21a","entry":"nonlinearity","repo":"zh460045050/v2l-tokenizer","repo_kind":"official","path":"models/encoder_decoder.py","file_url":"https://github.com/zh460045050/v2l-tokenizer/blob/HEAD/models/encoder_decoder.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3137073275f8c21a"}},{"code_sha256_prefix":"7257fdf155f5ef45","entry":"normalize_tensor","repo":"zh460045050/V2L-Tokenizer","repo_kind":"official","path":"models/models_v2l.py","file_url":"https://github.com/zh460045050/V2L-Tokenizer/blob/HEAD/models/models_v2l.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7257fdf155f5ef45"}},{"code_sha256_prefix":"053dbeb72d7622ac","entry":"preprocess_vqgan","repo":"zh460045050/v2l-tokenizer","repo_kind":"official","path":"step4_training_v2l_tokenizer.py","file_url":"https://github.com/zh460045050/v2l-tokenizer/blob/HEAD/step4_training_v2l_tokenizer.py","link_basis":"plan_row","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"053dbeb72d7622ac"}},{"code_sha256_prefix":"e268053216b1dd62","entry":"spatial_average","repo":"zh460045050/V2L-Tokenizer","repo_kind":"official","path":"models/models_v2l.py","file_url":"https://github.com/zh460045050/V2L-Tokenizer/blob/HEAD/models/models_v2l.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e268053216b1dd62"}},{"code_sha256_prefix":"87acba4e48e52920","entry":"vgg16","repo":"zh460045050/V2L-Tokenizer","repo_kind":"official","path":"models/models_v2l.py","file_url":"https://github.com/zh460045050/V2L-Tokenizer/blob/HEAD/models/models_v2l.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"87acba4e48e52920"}},{"code_sha256_prefix":"9542161e9640b858","entry":"whitespace_clean","repo":"zh460045050/v2l-tokenizer","repo_kind":"official","path":"models/global_encoder/simple_tokenizer.py","file_url":"https://github.com/zh460045050/v2l-tokenizer/blob/HEAD/models/global_encoder/simple_tokenizer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9542161e9640b858"}},{"code_sha256_prefix":"1271bf0010aa6cbd","entry":"Decoder","repo":"zh460045050/V2L-Tokenizer","repo_kind":"official","path":"models/models_v2l.py","file_url":"https://github.com/zh460045050/V2L-Tokenizer/blob/HEAD/models/models_v2l.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1271bf0010aa6cbd"}},{"code_sha256_prefix":"855e9decce73b852","entry":"LPIPS","repo":"zh460045050/V2L-Tokenizer","repo_kind":"official","path":"models/models_v2l.py","file_url":"https://github.com/zh460045050/V2L-Tokenizer/blob/HEAD/models/models_v2l.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"855e9decce73b852"}},{"code_sha256_prefix":"a30fee39c40fb5d1","entry":"NLayerDiscriminator","repo":"zh460045050/V2L-Tokenizer","repo_kind":"official","path":"models/models_v2l.py","file_url":"https://github.com/zh460045050/V2L-Tokenizer/blob/HEAD/models/models_v2l.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a30fee39c40fb5d1"}},{"code_sha256_prefix":"48df3c8dc5bfafc3","entry":"NetLinLayer","repo":"zh460045050/V2L-Tokenizer","repo_kind":"official","path":"models/models_v2l.py","file_url":"https://github.com/zh460045050/V2L-Tokenizer/blob/HEAD/models/models_v2l.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"48df3c8dc5bfafc3"}},{"code_sha256_prefix":"769f6ff8ef7da5e8","entry":"VQModel_LLaMA","repo":"zh460045050/V2L-Tokenizer","repo_kind":"official","path":"models/models_v2l.py","file_url":"https://github.com/zh460045050/V2L-Tokenizer/blob/HEAD/models/models_v2l.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"769f6ff8ef7da5e8"}},{"code_sha256_prefix":"0def50a351111624","entry":"calculate_frechet_distance","repo":"zh460045050/v2l-tokenizer","repo_kind":"official","path":"eval_denoising_generation.py","file_url":"https://github.com/zh460045050/v2l-tokenizer/blob/HEAD/eval_denoising_generation.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0def50a351111624"}},{"code_sha256_prefix":"24234ada3a1a42c0","entry":"download","repo":"zh460045050/V2L-Tokenizer","repo_kind":"official","path":"models/models_v2l.py","file_url":"https://github.com/zh460045050/V2L-Tokenizer/blob/HEAD/models/models_v2l.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"24234ada3a1a42c0"}},{"code_sha256_prefix":"f0ac78b4e05be194","entry":"instantiate_from_config","repo":"zh460045050/v2l-tokenizer","repo_kind":"official","path":"models/models_v2l.py","file_url":"https://github.com/zh460045050/v2l-tokenizer/blob/HEAD/models/models_v2l.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f0ac78b4e05be194"}},{"code_sha256_prefix":"34aa7b2ab98f397f","entry":"load_vqgan","repo":"zh460045050/v2l-tokenizer","repo_kind":"official","path":"step4_training_v2l_tokenizer.py","file_url":"https://github.com/zh460045050/v2l-tokenizer/blob/HEAD/step4_training_v2l_tokenizer.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"34aa7b2ab98f397f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}