{"url":"/task/lip-reading","name":"Lip Reading","slug":"lip-reading","description_markdown":"**Lip Reading** is a task to infer the speech content in a video by using only the visual information, especially the lip movements. It has many crucial applications in practice, such as assisting audio-based speech recognition, biometric authentication and aiding hearing-impaired people.\n\n\n<span class=\"description-source\">Source: [Mutual Information Maximization for Effective Lip Reading ](https://arxiv.org/abs/2003.06439)</span>","categories":[{"name":"Time Series","url":"/area/time-series"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":153,"papers_with_code":49,"benchmarks":3,"benchmark_tables_in_archive":3,"benchmark_tables_shown":3,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":6,"subtasks":1,"parent_tasks":0},"benchmarks":[{"leaderboard":"/sota/lip-reading-on-grid-corpus-mixed-speech","slug":"lip-reading-on-grid-corpus-mixed-speech","dataset":"GRID corpus (mixed-speech)","dataset_url":"/dataset/grid","rows_in_archive":1,"metrics":["WER"],"first_row_in_archive_order":{"model":"Lip2Wav","paper_title":"Learning Individual Speaking Styles for Accurate Lip to Speech Synthesis","paper_url":"/paper/learning-individual-speaking-styles-for","paper_date":"2020-05-17","arxiv_id":"2005.08209","code_links":[{"title":"Rudrabha/Lip2Wav","url":"https://github.com/Rudrabha/Lip2Wav"}],"syntology":{"n":9,"n_ran":1,"n_unverified":8,"n_pointer_only":0}}},{"leaderboard":"/sota/lip-reading-on-lrw","slug":"lip-reading-on-lrw","dataset":"LRW","dataset_url":"/dataset/lrw","rows_in_archive":1,"metrics":["WER"],"first_row_in_archive_order":{"model":"Lip2Wav","paper_title":"Learning Individual Speaking Styles for Accurate Lip to Speech Synthesis","paper_url":"/paper/learning-individual-speaking-styles-for","paper_date":"2020-05-17","arxiv_id":"2005.08209","code_links":[{"title":"Rudrabha/Lip2Wav","url":"https://github.com/Rudrabha/Lip2Wav"}],"syntology":{"n":9,"n_ran":1,"n_unverified":8,"n_pointer_only":0}}},{"leaderboard":"/sota/lip-reading-on-tcd-timit-corpus-mixed-speech","slug":"lip-reading-on-tcd-timit-corpus-mixed-speech","dataset":"TCD-TIMIT corpus (mixed-speech)","dataset_url":"/dataset/timit","rows_in_archive":1,"metrics":["WER"],"first_row_in_archive_order":{"model":"Lip2Wav","paper_title":"Learning Individual Speaking Styles for Accurate Lip to Speech Synthesis","paper_url":"/paper/learning-individual-speaking-styles-for","paper_date":"2020-05-17","arxiv_id":"2005.08209","code_links":[{"title":"Rudrabha/Lip2Wav","url":"https://github.com/Rudrabha/Lip2Wav"}],"syntology":{"n":9,"n_ran":1,"n_unverified":8,"n_pointer_only":0}}}],"datasets":[{"url":"/dataset/lrw","name":"LRW","full_name":"Lip Reading in the Wild","num_papers_in_archive":188},{"url":"/dataset/timit","name":"TIMIT","full_name":"TIMIT Acoustic-Phonetic Continuous Speech Corpus","num_papers_in_archive":31},{"url":"/dataset/grid","name":"GRID Dataset","full_name":"","num_papers_in_archive":10},{"url":"/dataset/lrw-1000","name":"CAS-VSR-W1k (LRW-1000)","full_name":"CAS-VSR-W1k (LRW-1000)","num_papers_in_archive":9},{"url":"/dataset/glips","name":"GLips","full_name":"German Lips","num_papers_in_archive":6},{"url":"/dataset/cas-vsr-s101","name":"CAS-VSR-S101","full_name":"","num_papers_in_archive":1}],"subtasks":[{"url":"/task/lip-password-classification","name":"Lip password classification"}],"parent_tasks":[],"papers":{"order":"repositories listed in the archive (desc), then date (desc); the archive holds no stars","population":"papers tagged with this task that list at least one repository in the archive","shown":30,"of":49,"tagged_in_all":153,"items":[{"url":"/paper/deep-audio-visual-speech-recognition","title":"Deep Audio-Visual Speech Recognition","date":"2018-09-06","arxiv_id":"1809.02108","repositories_listed":4,"syntology":null},{"url":"/paper/combining-residual-networks-with-lstms-for","title":"Combining Residual Networks with LSTMs for Lipreading","date":"2017-03-12","arxiv_id":"1703.04105","repositories_listed":4,"syntology":{"n":2,"n_ran":2,"n_unverified":0,"n_pointer_only":1}},{"url":"/paper/end-to-end-audio-visual-speech-recognition","title":"End-to-end Audio-visual Speech Recognition with Conformers","date":"2021-02-12","arxiv_id":"2102.06657","repositories_listed":3,"syntology":null},{"url":"/paper/mixspeech-cross-modality-self-learning-with","title":"MixSpeech: Cross-Modality Self-Learning with Audio-Visual Stream Mixup for Visual Speech Translation and Recognition","date":"2023-03-09","arxiv_id":"2303.05309","repositories_listed":2,"syntology":{"n":7,"n_ran":7,"n_unverified":0,"n_pointer_only":0}},{"url":"/paper/learning-audio-visual-speech-representation-1","title":"Learning Audio-Visual Speech Representation by Masked Multimodal Cluster Prediction","date":"2022-01-05","arxiv_id":"2201.02184","repositories_listed":2,"syntology":null},{"url":"/paper/authnet-a-deep-learning-based-authentication","title":"AuthNet: A Deep Learning based Authentication Mechanism using Temporal Facial Feature Movements","date":"2020-12-04","arxiv_id":"2012.02515","repositories_listed":2,"syntology":null},{"url":"/paper/lipreading-using-temporal-convolutional","title":"Lipreading using Temporal Convolutional Networks","date":"2020-01-23","arxiv_id":"2001.08702","repositories_listed":2,"syntology":{"n":2,"n_ran":1,"n_unverified":1,"n_pointer_only":2}},{"url":"/paper/lrw-1000-a-naturally-distributed-large-scale","title":"LRW-1000: A Naturally-Distributed Large-Scale Benchmark for Lip Reading in the Wild","date":"2018-10-16","arxiv_id":"1810.06990","repositories_listed":2,"syntology":null},{"url":"/paper/transforming-faces-into-video-stories","title":"Transforming faces into video stories -- VideoFace2.0","date":"2025-05-04","arxiv_id":"2505.02060","repositories_listed":1,"syntology":null},{"url":"/paper/lend-a-hand-semi-training-free-cued-speech","title":"Lend a Hand: Semi Training-Free Cued Speech Recognition via MLLM-Driven Hand Modeling for Barrier-free Communication","date":"2025-03-11","arxiv_id":"2503.21785","repositories_listed":1,"syntology":null},{"url":"/paper/personalized-lip-reading-adapting-to-your","title":"Personalized Lip Reading: Adapting to Your Unique Lip Movements with Vision and Language","date":"2024-09-02","arxiv_id":"2409.00986","repositories_listed":1,"syntology":null},{"url":"/paper/audio-visual-speech-recognition-based-on","title":"Audio-Visual Speech Recognition based on Regulated Transformer and Spatio-Temporal Fusion Strategy for Driver Assistive Systems","date":"2024-05-09","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/mtga-multi-view-temporal-granularity-aligned","title":"MTGA: Multi-View Temporal Granularity Aligned Aggregation for Event-Based Lip-Reading","date":"2024-04-18","arxiv_id":"2404.11979","repositories_listed":1,"syntology":null},{"url":"/paper/where-visual-speech-meets-language-vsp-llm","title":"Where Visual Speech Meets Language: VSP-LLM Framework for Efficient and Context-Aware Visual Speech Processing","date":"2024-02-23","arxiv_id":"2402.15151","repositories_listed":1,"syntology":{"n":12,"n_ran":8,"n_unverified":4,"n_pointer_only":12}},{"url":"/paper/neural-text-to-articulate-talk-deep-text-to","title":"Neural Text to Articulate Talk: Deep Text to Audiovisual Speech Synthesis achieving both Auditory and Photo-realism","date":"2023-12-11","arxiv_id":"2312.06613","repositories_listed":1,"syntology":null},{"url":"/paper/do-vsr-models-generalize-beyond-lrs3","title":"Do VSR Models Generalize Beyond LRS3?","date":"2023-11-23","arxiv_id":"2311.14063","repositories_listed":1,"syntology":null},{"url":"/paper/learning-separable-hidden-unit-contributions","title":"Learning Separable Hidden Unit Contributions for Speaker-Adaptive Lip-Reading","date":"2023-10-08","arxiv_id":"2310.05058","repositories_listed":1,"syntology":null},{"url":"/paper/selftalk-a-self-supervised-commutative","title":"SelfTalk: A Self-Supervised Commutative Training Diagram to Comprehend 3D Talking Faces","date":"2023-06-19","arxiv_id":"2306.10799","repositories_listed":1,"syntology":{"n":3,"n_ran":3,"n_unverified":0,"n_pointer_only":3}},{"url":"/paper/opensr-open-modality-speech-recognition-via","title":"OpenSR: Open-Modality Speech Recognition via Maintaining Multi-Modality Alignment","date":"2023-06-10","arxiv_id":"2306.06410","repositories_listed":1,"syntology":null},{"url":"/paper/a-novel-interpretable-and-generalizable-re","title":"A Novel Interpretable and Generalizable Re-synchronization Model for Cued Speech based on a Multi-Cuer Corpus","date":"2023-06-05","arxiv_id":"2306.02596","repositories_listed":1,"syntology":null},{"url":"/paper/lipvoicer-generating-speech-from-silent","title":"LipVoicer: Generating Speech from Silent Videos Guided by Lip Reading","date":"2023-06-05","arxiv_id":"2306.03258","repositories_listed":1,"syntology":{"n":8,"n_ran":5,"n_unverified":3,"n_pointer_only":0}},{"url":"/paper/seeing-what-you-said-talking-face-generation","title":"Seeing What You Said: Talking Face Generation Guided by a Lip Reading Expert","date":"2023-03-29","arxiv_id":"2303.17480","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_unverified":0,"n_pointer_only":1}},{"url":"/paper/geneface-generalized-and-high-fidelity-audio","title":"GeneFace: Generalized and High-Fidelity Audio-Driven 3D Talking Face Synthesis","date":"2023-01-31","arxiv_id":"2301.13430","repositories_listed":1,"syntology":{"n":4,"n_ran":3,"n_unverified":1,"n_pointer_only":0}},{"url":"/paper/olkavs-an-open-large-scale-korean-audio","title":"OLKAVS: An Open Large-Scale Korean Audio-Visual Speech Dataset","date":"2023-01-16","arxiv_id":"2301.06375","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_unverified":0,"n_pointer_only":2}},{"url":"/paper/audio-visual-efficient-conformer-for-robust","title":"Audio-Visual Efficient Conformer for Robust Speech Recognition","date":"2023-01-04","arxiv_id":"2301.01456","repositories_listed":1,"syntology":{"n":6,"n_ran":0,"n_unverified":6,"n_pointer_only":0}},{"url":"/paper/lip-sync-matters-a-novel-multimodal-forgery","title":"Lip Sync Matters: A Novel Multimodal Forgery Detector","date":"2022-11-07","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/relaxed-attention-for-transformer-models","title":"Relaxed Attention for Transformer Models","date":"2022-09-20","arxiv_id":"2209.09735","repositories_listed":1,"syntology":null},{"url":"/paper/training-strategies-for-improved-lip-reading","title":"Training Strategies for Improved Lip-reading","date":"2022-09-03","arxiv_id":"2209.01383","repositories_listed":1,"syntology":null},{"url":"/paper/speaker-adaptive-lip-reading-with-user","title":"Speaker-adaptive Lip Reading with User-dependent Padding","date":"2022-08-09","arxiv_id":"2208.04498","repositories_listed":1,"syntology":null},{"url":"/paper/multi-modality-associative-bridging-through-1","title":"Multi-modality Associative Bridging through Memory: Speech Sound Recollected from Face Video","date":"2022-04-04","arxiv_id":"2204.01265","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_unverified":0,"n_pointer_only":1}}],"syntology_records":11,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-09-24T18:15:14+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"}},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"}}