{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/lip2vec-efficient-and-robust-visual-speech","title":"Lip2Vec: Efficient and Robust Visual Speech Recognition via Latent-to-Latent Visual to Audio Representation Mapping","arxiv_id":"2308.06112","date":"2023-08-11","proceeding":"ICCV 2023 1","authors":["Yasser Abdelaziz Dahou Djilali","Sanath Narayan","Haithem Boussaid","Ebtessam Almazrouei","Merouane Debbah"],"abstract":"Visual Speech Recognition (VSR) differs from the common perception tasks as it requires deeper reasoning over the video sequence, even by human experts. Despite the recent advances in VSR, current approaches rely on labeled data to fully train or finetune their models predicting the target speech. This hinders their ability to generalize well beyond the training set and leads to performance degeneration under out-of-distribution challenging scenarios. Unlike previous works that involve auxiliary losses or complex training procedures and architectures, we propose a simple approach, named Lip2Vec that is based on learning a prior model. Given a robust visual speech encoder, this network maps the encoded latent representations of the lip sequence to their corresponding latents from the audio pair, which are sufficiently invariant for effective text decoding. The generated audio representation is then decoded to text using an off-the-shelf Audio Speech Recognition (ASR) model. The proposed model compares favorably with fully-supervised learning methods on the LRS3 dataset achieving 26 WER. Unlike SoTA approaches, our model keeps a reasonable performance on the VoxCeleb test set. We believe that reprogramming the VSR as an ASR task narrows the performance gap between the two and paves the way for more flexible formulations of lip reading.","url_abs":"https://arxiv.org/abs/2308.06112v1","url_pdf":"https://arxiv.org/pdf/2308.06112v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[],"tasks":[{"task_slug":"lip-reading","task_name":"Lip Reading"},{"task_slug":"speech-recognition","task_name":"Speech Recognition"},{"task_slug":"visual-speech-recognition","task_name":"Visual Speech Recognition"},{"task_slug":"speech-recognition-1","task_name":"speech-recognition"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2308.06112","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2308.06112"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/YasserdahouML/Lip2Vec","reach":{"status":"ok"}}],"summary":{"ran":11,"ran_honours":1,"unverified":7},"by_repo_kind":{"found_in_text":{"samples":19,"ran":12,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":19,"samples":[{"code_sha256_prefix":"362b232acbc220b7","entry":"apply_transform","repo":"YasserdahouML/Lip2Vec","repo_kind":"found_in_text","path":"datasets/preprocess.py","file_url":"https://github.com/YasserdahouML/Lip2Vec/blob/HEAD/datasets/preprocess.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"362b232acbc220b7"}},{"code_sha256_prefix":"c6fb86dd68007249","entry":"crop","repo":"YasserdahouML/Lip2Vec","repo_kind":"found_in_text","path":"datasets/functional_video.py","file_url":"https://github.com/YasserdahouML/Lip2Vec/blob/HEAD/datasets/functional_video.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c6fb86dd68007249"}},{"code_sha256_prefix":"6e0808fc4828513a","entry":"exact_div","repo":"YasserdahouML/Lip2Vec","repo_kind":"found_in_text","path":"datasets/utils.py","file_url":"https://github.com/YasserdahouML/Lip2Vec/blob/HEAD/datasets/utils.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6e0808fc4828513a"}},{"code_sha256_prefix":"5fcb88dca39140d4","entry":"get_cer","repo":"YasserdahouML/Lip2Vec","repo_kind":"found_in_text","path":"metrics/measures.py","file_url":"https://github.com/YasserdahouML/Lip2Vec/blob/HEAD/metrics/measures.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5fcb88dca39140d4"}},{"code_sha256_prefix":"8b03deae3bdc3ad3","entry":"get_video_properties","repo":"YasserdahouML/Lip2Vec","repo_kind":"found_in_text","path":"datasets/utils_data.py","file_url":"https://github.com/YasserdahouML/Lip2Vec/blob/HEAD/datasets/utils_data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8b03deae3bdc3ad3"}},{"code_sha256_prefix":"5fc5cf4404249a10","entry":"linear_interpolate","repo":"YasserdahouML/Lip2Vec","repo_kind":"found_in_text","path":"datasets/preprocess.py","file_url":"https://github.com/YasserdahouML/Lip2Vec/blob/HEAD/datasets/preprocess.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5fc5cf4404249a10"}},{"code_sha256_prefix":"b3510aa8b8ed8518","entry":"load_audio","repo":"YasserdahouML/Lip2Vec","repo_kind":"found_in_text","path":"datasets/utils_data.py","file_url":"https://github.com/YasserdahouML/Lip2Vec/blob/HEAD/datasets/utils_data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b3510aa8b8ed8518"}},{"code_sha256_prefix":"8c9319bbaa8f67c1","entry":"read_txt_lines","repo":"YasserdahouML/Lip2Vec","repo_kind":"found_in_text","path":"datasets/utils_data.py","file_url":"https://github.com/YasserdahouML/Lip2Vec/blob/HEAD/datasets/utils_data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8c9319bbaa8f67c1"}},{"code_sha256_prefix":"a42f16eadfb7718f","entry":"resize","repo":"YasserdahouML/Lip2Vec","repo_kind":"found_in_text","path":"datasets/functional_video.py","file_url":"https://github.com/YasserdahouML/Lip2Vec/blob/HEAD/datasets/functional_video.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a42f16eadfb7718f"}},{"code_sha256_prefix":"b0239305ba36c73f","entry":"resized_crop","repo":"YasserdahouML/Lip2Vec","repo_kind":"found_in_text","path":"datasets/functional_video.py","file_url":"https://github.com/YasserdahouML/Lip2Vec/blob/HEAD/datasets/functional_video.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b0239305ba36c73f"}},{"code_sha256_prefix":"a314b3fcba001058","entry":"sinusoids","repo":"YasserdahouML/Lip2Vec","repo_kind":"found_in_text","path":"models/prior.py","file_url":"https://github.com/YasserdahouML/Lip2Vec/blob/HEAD/models/prior.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a314b3fcba001058"}},{"code_sha256_prefix":"353e88e37719cd08","entry":"warp_img","repo":"YasserdahouML/Lip2Vec","repo_kind":"found_in_text","path":"datasets/preprocess.py","file_url":"https://github.com/YasserdahouML/Lip2Vec/blob/HEAD/datasets/preprocess.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"353e88e37719cd08"}},{"code_sha256_prefix":"5c86d34cae9ef6a1","entry":"build_transformer","repo":"YasserdahouML/Lip2Vec","repo_kind":"found_in_text","path":"models/transformer.py","file_url":"https://github.com/YasserdahouML/Lip2Vec/blob/HEAD/models/transformer.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5c86d34cae9ef6a1"}},{"code_sha256_prefix":"f646df9b796b4337","entry":"get_er","repo":"YasserdahouML/Lip2Vec","repo_kind":"found_in_text","path":"metrics/measures.py","file_url":"https://github.com/YasserdahouML/Lip2Vec/blob/HEAD/metrics/measures.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f646df9b796b4337"}},{"code_sha256_prefix":"f0ad236b3b39fdaf","entry":"get_wer","repo":"YasserdahouML/Lip2Vec","repo_kind":"found_in_text","path":"metrics/measures.py","file_url":"https://github.com/YasserdahouML/Lip2Vec/blob/HEAD/metrics/measures.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f0ad236b3b39fdaf"}},{"code_sha256_prefix":"edf924293b4b2375","entry":"mel_filters","repo":"YasserdahouML/Lip2Vec","repo_kind":"found_in_text","path":"datasets/audio.py","file_url":"https://github.com/YasserdahouML/Lip2Vec/blob/HEAD/datasets/audio.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"edf924293b4b2375"}},{"code_sha256_prefix":"e56996b20b9c19cb","entry":"optional_int","repo":"YasserdahouML/Lip2Vec","repo_kind":"found_in_text","path":"datasets/utils.py","file_url":"https://github.com/YasserdahouML/Lip2Vec/blob/HEAD/datasets/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e56996b20b9c19cb"}},{"code_sha256_prefix":"9f13de1c23e4003d","entry":"pad_or_trim","repo":"YasserdahouML/Lip2Vec","repo_kind":"found_in_text","path":"datasets/audio.py","file_url":"https://github.com/YasserdahouML/Lip2Vec/blob/HEAD/datasets/audio.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9f13de1c23e4003d"}},{"code_sha256_prefix":"176b6ba6c2c2940c","entry":"str2bool","repo":"YasserdahouML/Lip2Vec","repo_kind":"found_in_text","path":"datasets/utils.py","file_url":"https://github.com/YasserdahouML/Lip2Vec/blob/HEAD/datasets/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"176b6ba6c2c2940c"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}