{"url":"/task/visual-speech-recognition","name":"Visual Speech Recognition","slug":"visual-speech-recognition","description_markdown":null,"categories":[{"name":"Computer Vision","url":"/area/computer-vision"},{"name":"Music","url":"/area/music"},{"name":"Speech","url":"/area/speech"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":182,"papers_with_code":62,"benchmarks":2,"benchmark_tables_in_archive":2,"benchmark_tables_shown":2,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":6,"subtasks":1,"parent_tasks":1},"benchmarks":[{"leaderboard":"/sota/visual-speech-recognition-on-lrs3-ted","slug":"visual-speech-recognition-on-lrs3-ted","dataset":"LRS3-TED","dataset_url":"/dataset/lrs3-ted","rows_in_archive":3,"metrics":["Word Error Rate (WER)"],"first_row_in_archive_order":{"model":"CTC/Attention","paper_title":"Auto-AVSR: Audio-Visual Speech Recognition with Automatic Labels","paper_url":"/paper/auto-avsr-audio-visual-speech-recognition","paper_date":"2023-03-25","arxiv_id":"2303.14307","code_links":[{"title":"mpc001/auto_avsr","url":"https://github.com/mpc001/auto_avsr"},{"title":"umbertocappellazzo/llama-avsr","url":"https://github.com/umbertocappellazzo/llama-avsr"}],"syntology":{"n":6,"n_ran":0,"n_unverified":6,"n_pointer_only":0}}},{"leaderboard":"/sota/visual-speech-recognition-on-lrs2","slug":"visual-speech-recognition-on-lrs2","dataset":"LRS2","dataset_url":"/dataset/lrs2","rows_in_archive":2,"metrics":["Word Error Rate (WER)"],"first_row_in_archive_order":{"model":"VTP with more data","paper_title":"Sub-word Level Lip Reading With Visual Attention","paper_url":"/paper/sub-word-level-lip-reading-with-visual","paper_date":"2021-10-14","arxiv_id":"2110.07603","code_links":[],"syntology":null}}],"datasets":[{"url":"/dataset/lrs2","name":"LRS2","full_name":"Lip Reading Sentences 2","num_papers_in_archive":115},{"url":"/dataset/lrs3-ted","name":"LRS3-TED","full_name":"","num_papers_in_archive":63},{"url":"/dataset/lrw-1000","name":"CAS-VSR-W1k (LRW-1000)","full_name":"CAS-VSR-W1k (LRW-1000)","num_papers_in_archive":9},{"url":"/dataset/glips","name":"GLips","full_name":"German Lips","num_papers_in_archive":6},{"url":"/dataset/av-digits-database","name":"AV Digits Database","full_name":"AV Digits Database","num_papers_in_archive":2},{"url":"/dataset/watch-your-mouth-point-clouds-based-speech","name":"Watch Your Mouth: Point Clouds based Speech Recognition Dataset","full_name":"","num_papers_in_archive":1}],"subtasks":[{"url":"/task/lip-to-speech-synthesis","name":"Lip to Speech Synthesis"}],"parent_tasks":[{"url":"/task/speech-recognition","name":"Speech Recognition"}],"papers":{"order":"repositories listed in the archive (desc), then date (desc); the archive holds no stars","population":"papers tagged with this task that list at least one repository in the archive","shown":30,"of":62,"tagged_in_all":182,"items":[{"url":"/paper/deep-audio-visual-speech-recognition","title":"Deep Audio-Visual Speech Recognition","date":"2018-09-06","arxiv_id":"1809.02108","repositories_listed":4,"syntology":null},{"url":"/paper/combining-residual-networks-with-lstms-for","title":"Combining Residual Networks with LSTMs for Lipreading","date":"2017-03-12","arxiv_id":"1703.04105","repositories_listed":4,"syntology":{"n":2,"n_ran":2,"n_unverified":0,"n_pointer_only":1}},{"url":"/paper/end-to-end-audio-visual-speech-recognition","title":"End-to-end Audio-visual Speech Recognition with Conformers","date":"2021-02-12","arxiv_id":"2102.06657","repositories_listed":3,"syntology":null},{"url":"/paper/the-npu-aslp-liauto-system-description-for","title":"The NPU-ASLP-LiAuto System Description for Visual Speech Recognition in CNVSRC 2023","date":"2024-01-07","arxiv_id":"2401.06788","repositories_listed":2,"syntology":null},{"url":"/paper/auto-avsr-audio-visual-speech-recognition","title":"Auto-AVSR: Audio-Visual Speech Recognition with Automatic Labels","date":"2023-03-25","arxiv_id":"2303.14307","repositories_listed":2,"syntology":{"n":6,"n_ran":0,"n_unverified":6,"n_pointer_only":0}},{"url":"/paper/mixspeech-cross-modality-self-learning-with","title":"MixSpeech: Cross-Modality Self-Learning with Audio-Visual Stream Mixup for Visual Speech Translation and Recognition","date":"2023-03-09","arxiv_id":"2303.05309","repositories_listed":2,"syntology":{"n":7,"n_ran":7,"n_unverified":0,"n_pointer_only":0}},{"url":"/paper/visual-speech-recognition-for-multiple","title":"Visual Speech Recognition for Multiple Languages in the Wild","date":"2022-02-26","arxiv_id":"2202.13084","repositories_listed":2,"syntology":null},{"url":"/paper/lrw-1000-a-naturally-distributed-large-scale","title":"LRW-1000: A Naturally-Distributed Large-Scale Benchmark for Lip Reading in the Wild","date":"2018-10-16","arxiv_id":"1810.06990","repositories_listed":2,"syntology":null},{"url":"/paper/transfer-learning-from-visual-speech","title":"Transfer Learning from Visual Speech Recognition to Mouthing Recognition in German Sign Language","date":"2025-05-20","arxiv_id":"2505.13784","repositories_listed":1,"syntology":null},{"url":"/paper/cogenav-versatile-audio-visual-representation","title":"CoGenAV: Versatile Audio-Visual Representation Learning via Contrastive-Generative Synchronization","date":"2025-05-06","arxiv_id":"2505.03186","repositories_listed":1,"syntology":null},{"url":"/paper/mms-llama-efficient-llm-based-audio-visual-1","title":"MMS-LLaMA: Efficient LLM-based Audio-Visual Speech Recognition with Minimal Multimodal Speech Tokens","date":"2025-03-14","arxiv_id":"2503.11315","repositories_listed":1,"syntology":null},{"url":"/paper/zero-avsr-zero-shot-audio-visual-speech","title":"Zero-AVSR: Zero-Shot Audio-Visual Speech Recognition with LLMs by Learning Language-Agnostic Speech Representations","date":"2025-03-08","arxiv_id":"2503.06273","repositories_listed":1,"syntology":null},{"url":"/paper/audio-visual-representation-learning-via","title":"Audio-Visual Representation Learning via Knowledge Distillation from Speech Foundation Models","date":"2025-02-09","arxiv_id":"2502.05766","repositories_listed":1,"syntology":null},{"url":"/paper/mwhisper-flamingo-for-multilingual-audio","title":"mWhisper-Flamingo for Multilingual Audio-Visual Noise-Robust Speech Recognition","date":"2025-02-03","arxiv_id":"2502.01547","repositories_listed":1,"syntology":null},{"url":"/paper/evaluation-of-end-to-end-continuous-spanish","title":"Evaluation of End-to-End Continuous Spanish Lipreading in Different Data Conditions","date":"2025-02-01","arxiv_id":"2502.00464","repositories_listed":1,"syntology":null},{"url":"/paper/multi-task-corrupted-prediction-for-learning","title":"Multi-Task Corrupted Prediction for Learning Robust Audio-Visual Speech Representation","date":"2025-01-23","arxiv_id":"2504.18539","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_unverified":0,"n_pointer_only":0}},{"url":"/paper/listening-and-seeing-again-generative-error","title":"Listening and Seeing Again: Generative Error Correction for Audio-Visual Speech Recognition","date":"2025-01-03","arxiv_id":"2501.04038","repositories_listed":1,"syntology":null},{"url":"/paper/alignvsr-audio-visual-cross-modal-alignment","title":"AlignVSR: Audio-Visual Cross-Modal Alignment for Visual Speech Recognition","date":"2024-10-21","arxiv_id":"2410.16438","repositories_listed":1,"syntology":null},{"url":"/paper/large-language-models-are-strong-audio-visual","title":"Large Language Models are Strong Audio-Visual Speech Recognition Learners","date":"2024-09-18","arxiv_id":"2409.12319","repositories_listed":1,"syntology":{"n":12,"n_ran":9,"n_unverified":3,"n_pointer_only":12}},{"url":"/paper/2408-02369","title":"The NPU-ASLP System Description for Visual Speech Recognition in CNVSRC 2024","date":"2024-08-05","arxiv_id":"2408.02369","repositories_listed":1,"syntology":null},{"url":"/paper/2408-00624","title":"SynesLM: A Unified Approach for Audio-visual Speech Recognition and Translation via Language Model and Synthetic Data","date":"2024-08-01","arxiv_id":"2408.00624","repositories_listed":1,"syntology":null},{"url":"/paper/tailored-design-of-audio-visual-speech","title":"Tailored Design of Audio-Visual Speech Recognition Models using Branchformers","date":"2024-07-09","arxiv_id":"2407.06606","repositories_listed":1,"syntology":null},{"url":"/paper/learning-video-temporal-dynamics-with-cross","title":"Learning Video Temporal Dynamics with Cross-Modal Attention for Robust Audio-Visual Speech Recognition","date":"2024-07-04","arxiv_id":"2407.03563","repositories_listed":1,"syntology":{"n":13,"n_ran":10,"n_unverified":3,"n_pointer_only":0}},{"url":"/paper/syncvsr-data-efficient-visual-speech","title":"SyncVSR: Data-Efficient Visual Speech Recognition with End-to-End Crossmodal Audio Token Synchronization","date":"2024-06-18","arxiv_id":"2406.12233","repositories_listed":1,"syntology":null},{"url":"/paper/whisper-flamingo-integrating-visual-features","title":"Whisper-Flamingo: Integrating Visual Features into Whisper for Audio-Visual Speech Recognition and Translation","date":"2024-06-14","arxiv_id":"2406.10082","repositories_listed":1,"syntology":{"n":18,"n_ran":5,"n_unverified":13,"n_pointer_only":18}},{"url":"/paper/watch-your-mouth-silent-speech-recognition","title":"Watch Your Mouth: Silent Speech Recognition with Depth Sensing","date":"2024-05-11","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/audio-visual-speech-recognition-based-on","title":"Audio-Visual Speech Recognition based on Regulated Transformer and Spatio-Temporal Fusion Strategy for Driver Assistive Systems","date":"2024-05-09","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/a-study-of-dropout-induced-modality-bias-on","title":"A Study of Dropout-Induced Modality Bias on Robustness to Missing Video Frames for Audio-Visual Speech Recognition","date":"2024-03-07","arxiv_id":"2403.04245","repositories_listed":1,"syntology":null},{"url":"/paper/where-visual-speech-meets-language-vsp-llm","title":"Where Visual Speech Meets Language: VSP-LLM Framework for Efficient and Context-Aware Visual Speech Processing","date":"2024-02-23","arxiv_id":"2402.15151","repositories_listed":1,"syntology":{"n":12,"n_ran":8,"n_unverified":4,"n_pointer_only":12}},{"url":"/paper/it-s-never-too-late-fusing-acoustic","title":"It's Never Too Late: Fusing Acoustic Information into Large Language Models for Automatic Speech Recognition","date":"2024-02-08","arxiv_id":"2402.05457","repositories_listed":1,"syntology":{"n":6,"n_ran":2,"n_unverified":4,"n_pointer_only":0}}],"syntology_records":9,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-09-24T18:15:14+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"}},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"}}