{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/learning-individual-speaking-styles-for","title":"Learning Individual Speaking Styles for Accurate Lip to Speech Synthesis","arxiv_id":"2005.08209","date":"2020-05-17","proceeding":"CVPR 2020 6","authors":["K R Prajwal","Rudrabha Mukhopadhyay","Vinay Namboodiri","C. V. Jawahar"],"abstract":"Humans involuntarily tend to infer parts of the conversation from lip movements when the speech is absent or corrupted by external noise. In this work, we explore the task of lip to speech synthesis, i.e., learning to generate natural speech given only the lip movements of a speaker. Acknowledging the importance of contextual and speaker-specific cues for accurate lip-reading, we take a different path from existing works. We focus on learning accurate lip sequences to speech mappings for individual speakers in unconstrained, large vocabulary settings. To this end, we collect and release a large-scale benchmark dataset, the first of its kind, specifically to train and evaluate the single-speaker lip to speech task in natural settings. We propose a novel approach with key design choices to achieve accurate, natural lip to speech synthesis in such unconstrained scenarios for the first time. Extensive evaluation using quantitative, qualitative metrics and human evaluation shows that our method is four times more intelligible than previous works in this space. Please check out our demo video for a quick overview of the paper, method, and qualitative results. https://www.youtube.com/watch?v=HziA-jmlk_4&feature=youtu.be","url_abs":"https://arxiv.org/abs/2005.08209v1","url_pdf":"https://arxiv.org/pdf/2005.08209v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"learning-individual-speaking-styles-for","repo_url":"https://github.com/Rudrabha/Lip2Wav","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"lip-reading","task_name":"Lip Reading"},{"task_slug":"lip-to-speech-synthesis","task_name":"Lip to Speech Synthesis"},{"task_slug":"speaker-specific-lip-to-speech-synthesis","task_name":"Speaker-Specific Lip to Speech Synthesis"},{"task_slug":"speech-synthesis","task_name":"Speech Synthesis"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/lip-reading-on-grid-corpus-mixed-speech","task":"Lip Reading","dataset":"GRID corpus (mixed-speech)","model":"Lip2Wav","rank_in_archive_order":1,"of":1,"metrics":{"WER":"14.08"},"uses_additional_data":false},{"leaderboard":"/sota/lip-reading-on-lrw","task":"Lip Reading","dataset":"LRW","model":"Lip2Wav","rank_in_archive_order":1,"of":1,"metrics":{"WER":"34.2"},"uses_additional_data":false},{"leaderboard":"/sota/lip-reading-on-tcd-timit-corpus-mixed-speech","task":"Lip Reading","dataset":"TCD-TIMIT corpus (mixed-speech)","model":"Lip2Wav","rank_in_archive_order":1,"of":1,"metrics":{"WER":"31.26"},"uses_additional_data":false},{"leaderboard":"/sota/lip-to-speech-synthesis-on-lrw","task":"Lip to Speech Synthesis","dataset":"LRW","model":"Lip2Wav","rank_in_archive_order":1,"of":1,"metrics":{"ESTOI":"0.344","PESQ":"1.197","STOI":"0.543"},"uses_additional_data":false},{"leaderboard":"/sota/speaker-specific-lip-to-speech-synthesis-on","task":"Speaker-Specific Lip to Speech Synthesis","dataset":"GRID corpus (mixed-speech)","model":"Lip2Wav","rank_in_archive_order":2,"of":2,"metrics":{"ESTOI":"0.535","PESQ":"1.772","STOI":"0.731"},"uses_additional_data":false},{"leaderboard":"/sota/speaker-specific-lip-to-speech-synthesis-on-7","task":"Speaker-Specific Lip to Speech Synthesis","dataset":"Lip2Wav (Chem)","model":"Lip2Wav","rank_in_archive_order":2,"of":2,"metrics":{"ESTOI":"0.284","PESQ":"1.3","STOI":"0.416"},"uses_additional_data":false},{"leaderboard":"/sota/speaker-specific-lip-to-speech-synthesis-on-6","task":"Speaker-Specific Lip to Speech Synthesis","dataset":"Lip2Wav (Chess)","model":"Lip2Wav","rank_in_archive_order":2,"of":2,"metrics":{"ESTOI":"0.29","PESQ":"1.4","STOI":"0.418"},"uses_additional_data":false},{"leaderboard":"/sota/speaker-specific-lip-to-speech-synthesis-on-5","task":"Speaker-Specific Lip to Speech Synthesis","dataset":"Lip2Wav (DL)","model":"Lip2Wav","rank_in_archive_order":2,"of":2,"metrics":{"ESTOI":"0.183","PESQ":"1.671","STOI":"0.282"},"uses_additional_data":false},{"leaderboard":"/sota/speaker-specific-lip-to-speech-synthesis-on-3","task":"Speaker-Specific Lip to Speech Synthesis","dataset":"Lip2Wav (EH)","model":"Lip2Wav","rank_in_archive_order":2,"of":2,"metrics":{"ESTOI":"0.22","PESQ":"1.367","STOI":"0.369"},"uses_additional_data":false},{"leaderboard":"/sota/speaker-specific-lip-to-speech-synthesis-on-4","task":"Speaker-Specific Lip to Speech Synthesis","dataset":"Lip2Wav (HS)","model":"Lip2Wav","rank_in_archive_order":2,"of":2,"metrics":{"ESTOI":"0.311","PESQ":"1.29","STOI":"0.446"},"uses_additional_data":false},{"leaderboard":"/sota/speaker-specific-lip-to-speech-synthesis-on-1","task":"Speaker-Specific Lip to Speech Synthesis","dataset":"TCD-TIMIT corpus (mixed-speech)","model":"Lip2Wav","rank_in_archive_order":1,"of":1,"metrics":{"ESTOI":"36.5","PESQ":"1.35","STOI":"0.558"},"uses_additional_data":false}],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2005.08209","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.08209"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Rudrabha/Lip2Wav","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":1,"unverified":8},"by_repo_kind":{"official":{"samples":9,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"5baaa8c1b148ef70","entry":"conv3x3","repo":"Rudrabha/Lip2Wav","repo_kind":"official","path":"face_detection/models.py","file_url":"https://github.com/Rudrabha/Lip2Wav/blob/HEAD/face_detection/models.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5baaa8c1b148ef70"}},{"code_sha256_prefix":"535d1e92aecfff67","entry":"crop","repo":"Rudrabha/Lip2Wav","repo_kind":"official","path":"face_detection/utils.py","file_url":"https://github.com/Rudrabha/Lip2Wav/blob/HEAD/face_detection/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"535d1e92aecfff67"}},{"code_sha256_prefix":"a5424d7dbfd05689","entry":"crop_frame","repo":"Rudrabha/Lip2Wav","repo_kind":"official","path":"preprocess.py","file_url":"https://github.com/Rudrabha/Lip2Wav/blob/HEAD/preprocess.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a5424d7dbfd05689"}},{"code_sha256_prefix":"428bab6c5fe3195e","entry":"draw_gaussian","repo":"Rudrabha/Lip2Wav","repo_kind":"official","path":"face_detection/utils.py","file_url":"https://github.com/Rudrabha/Lip2Wav/blob/HEAD/face_detection/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"428bab6c5fe3195e"}},{"code_sha256_prefix":"8c52e380997759b5","entry":"get_image_list","repo":"Rudrabha/Lip2Wav","repo_kind":"official","path":"complete_test_generate.py","file_url":"https://github.com/Rudrabha/Lip2Wav/blob/HEAD/complete_test_generate.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8c52e380997759b5"}},{"code_sha256_prefix":"e69028e4a38f513d","entry":"get_image_list","repo":"Rudrabha/Lip2Wav","repo_kind":"official","path":"synthesizer/hparams.py","file_url":"https://github.com/Rudrabha/Lip2Wav/blob/HEAD/synthesizer/hparams.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e69028e4a38f513d"}},{"code_sha256_prefix":"bd89c559f716b290","entry":"inv_preemphasis","repo":"Rudrabha/Lip2Wav","repo_kind":"official","path":"synthesizer/audio.py","file_url":"https://github.com/Rudrabha/Lip2Wav/blob/HEAD/synthesizer/audio.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bd89c559f716b290"}},{"code_sha256_prefix":"56a333de6e89087a","entry":"preemphasis","repo":"Rudrabha/Lip2Wav","repo_kind":"official","path":"synthesizer/audio.py","file_url":"https://github.com/Rudrabha/Lip2Wav/blob/HEAD/synthesizer/audio.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"56a333de6e89087a"}},{"code_sha256_prefix":"fcc1fdc4b492b421","entry":"transform","repo":"Rudrabha/Lip2Wav","repo_kind":"official","path":"face_detection/utils.py","file_url":"https://github.com/Rudrabha/Lip2Wav/blob/HEAD/face_detection/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fcc1fdc4b492b421"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}