{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/where-visual-speech-meets-language-vsp-llm","title":"Where Visual Speech Meets Language: VSP-LLM Framework for Efficient and Context-Aware Visual Speech Processing","arxiv_id":"2402.15151","date":"2024-02-23","proceeding":null,"authors":["Jeong Hun Yeo","Seunghee Han","Minsu Kim","Yong Man Ro"],"abstract":"In visual speech processing, context modeling capability is one of the most important requirements due to the ambiguous nature of lip movements. For example, homophenes, words that share identical lip movements but produce different sounds, can be distinguished by considering the context. In this paper, we propose a novel framework, namely Visual Speech Processing incorporated with LLMs (VSP-LLM), to maximize the context modeling ability by bringing the overwhelming power of LLMs. Specifically, VSP-LLM is designed to perform multi-tasks of visual speech recognition and translation, where the given instructions control the type of task. The input video is mapped to the input latent space of an LLM by employing a self-supervised visual speech model. Focused on the fact that there is redundant information in input frames, we propose a novel deduplication method that reduces the embedded visual features by employing visual speech units. Through the proposed deduplication and Low Rank Adaptation (LoRA), VSP-LLM can be trained in a computationally efficient manner. In the translation dataset, the MuAViC benchmark, we demonstrate that VSP-LLM trained on just 30 hours of labeled data can more effectively translate lip movements compared to the recent model trained with 433 hours of data.","url_abs":"https://arxiv.org/abs/2402.15151v2","url_pdf":"https://arxiv.org/pdf/2402.15151v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"where-visual-speech-meets-language-vsp-llm","repo_url":"https://github.com/sally-sh/vsp-llm","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}}],"tasks":[{"task_slug":"lip-reading","task_name":"Lip Reading"},{"task_slug":"lipreading","task_name":"Lipreading"},{"task_slug":"speech-recognition","task_name":"Speech Recognition"},{"task_slug":"translation","task_name":"Translation"},{"task_slug":"visual-speech-recognition","task_name":"Visual Speech Recognition"},{"task_slug":"speech-recognition-1","task_name":"speech-recognition"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/lipreading-on-lrs3-ted","task":"Lipreading","dataset":"LRS3-TED","model":"VSP-LLM","rank_in_archive_order":7,"of":23,"metrics":{"Word Error Rate (WER)":"25.4"},"uses_additional_data":true}],"syntology":{"syntology_url":"https://syntology.ai/paper/2402.15151","atlas_url":"https://app.syntology.ai/?focus=2402.15151","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.15151"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sally-sh/vsp-llm","reach":{"status":"ok","spdx":"NOASSERTION"}}],"summary":{"ran_draft_wrong":3,"ran":5,"unverified":4},"by_repo_kind":{"official":{"samples":12,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":12,"samples":[{"code_sha256_prefix":"96cbb5e9ca5b6be0","entry":"Embedding","repo":"sally-sh/vsp-llm","repo_kind":"official","path":"src/vsp_llm.py","file_url":"https://github.com/sally-sh/vsp-llm/blob/HEAD/src/vsp_llm.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"96cbb5e9ca5b6be0"}},{"code_sha256_prefix":"8cd8cb0d1e9e63e4","entry":"Linear","repo":"sally-sh/vsp-llm","repo_kind":"official","path":"src/vsp_llm.py","file_url":"https://github.com/sally-sh/vsp-llm/blob/HEAD/src/vsp_llm.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8cd8cb0d1e9e63e4"}},{"code_sha256_prefix":"581de97f5d5aa97a","entry":"compute_mask_indices","repo":"sally-sh/vsp-llm","repo_kind":"official","path":"src/utils_vsp_llm.py","file_url":"https://github.com/sally-sh/vsp-llm/blob/HEAD/src/utils_vsp_llm.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"581de97f5d5aa97a"}},{"code_sha256_prefix":"583f9780bdd00a45","entry":"conv3x3","repo":"sally-sh/vsp-llm","repo_kind":"official","path":"src/resnet.py","file_url":"https://github.com/sally-sh/vsp-llm/blob/HEAD/src/resnet.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"583f9780bdd00a45"}},{"code_sha256_prefix":"d50e3a8f0b0f9a21","entry":"downsample_basic_block","repo":"sally-sh/vsp-llm","repo_kind":"official","path":"src/resnet.py","file_url":"https://github.com/sally-sh/vsp-llm/blob/HEAD/src/resnet.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d50e3a8f0b0f9a21"}},{"code_sha256_prefix":"546a770034e200d1","entry":"downsample_basic_block_v2","repo":"sally-sh/vsp-llm","repo_kind":"official","path":"src/resnet.py","file_url":"https://github.com/sally-sh/vsp-llm/blob/HEAD/src/resnet.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"546a770034e200d1"}},{"code_sha256_prefix":"db4a33e22cd9f644","entry":"find_runs","repo":"sally-sh/vsp-llm","repo_kind":"official","path":"src/utils_vsp_llm.py","file_url":"https://github.com/sally-sh/vsp-llm/blob/HEAD/src/utils_vsp_llm.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"db4a33e22cd9f644"}},{"code_sha256_prefix":"d5a7afb95212031e","entry":"load_video","repo":"sally-sh/vsp-llm","repo_kind":"official","path":"src/utils_vsp_llm.py","file_url":"https://github.com/sally-sh/vsp-llm/blob/HEAD/src/utils_vsp_llm.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d5a7afb95212031e"}},{"code_sha256_prefix":"f0641584d0fe0fb5","entry":"load_audio_visual","repo":"sally-sh/vsp-llm","repo_kind":"official","path":"src/hubert_dataset.py","file_url":"https://github.com/sally-sh/vsp-llm/blob/HEAD/src/hubert_dataset.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f0641584d0fe0fb5"}},{"code_sha256_prefix":"c9b8148d516d0989","entry":"load_audio_visual","repo":"sally-sh/vsp-llm","repo_kind":"official","path":"src/vsp_llm_dataset.py","file_url":"https://github.com/sally-sh/vsp-llm/blob/HEAD/src/vsp_llm_dataset.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"c9b8148d516d0989"}},{"code_sha256_prefix":"113a523619225ba0","entry":"load_label","repo":"sally-sh/vsp-llm","repo_kind":"official","path":"src/vsp_llm_dataset.py","file_url":"https://github.com/sally-sh/vsp-llm/blob/HEAD/src/vsp_llm_dataset.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"113a523619225ba0"}},{"code_sha256_prefix":"3e9450b6ac2fcddd","entry":"load_label_offset","repo":"sally-sh/vsp-llm","repo_kind":"official","path":"src/vsp_llm_dataset.py","file_url":"https://github.com/sally-sh/vsp-llm/blob/HEAD/src/vsp_llm_dataset.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3e9450b6ac2fcddd"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}