{"url":"/task/audio-visual-speech-recognition","name":"Audio-Visual Speech Recognition","slug":"audio-visual-speech-recognition","description_markdown":"Audio-visual speech recognition is the task of transcribing a paired audio and visual stream into text.","categories":[{"name":"Speech","url":"/area/speech"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":100,"papers_with_code":42,"benchmarks":4,"benchmark_tables_in_archive":4,"benchmark_tables_shown":4,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":7,"subtasks":1,"parent_tasks":0},"benchmarks":[{"leaderboard":"/sota/audio-visual-speech-recognition-on-lrs3-ted","slug":"audio-visual-speech-recognition-on-lrs3-ted","dataset":"LRS3-TED","dataset_url":"/dataset/lrs3-ted","rows_in_archive":12,"metrics":["Word Error Rate (WER)"],"first_row_in_archive_order":{"model":"MMS-LLaMA","paper_title":"MMS-LLaMA: Efficient LLM-based Audio-Visual Speech Recognition with Minimal Multimodal Speech Tokens","paper_url":"/paper/mms-llama-efficient-llm-based-audio-visual-1","paper_date":"2025-03-14","arxiv_id":"2503.11315","code_links":[{"title":"JeongHun0716/MMS-LLaMA","url":"https://github.com/JeongHun0716/MMS-LLaMA"}],"syntology":null}},{"leaderboard":"/sota/audio-visual-speech-recognition-on-lrs2","slug":"audio-visual-speech-recognition-on-lrs2","dataset":"LRS2","dataset_url":"/dataset/lrs2","rows_in_archive":8,"metrics":["Test WER"],"first_row_in_archive_order":{"model":"Whisper-Flamingo","paper_title":"Whisper-Flamingo: Integrating Visual Features into Whisper for Audio-Visual Speech Recognition and Translation","paper_url":"/paper/whisper-flamingo-integrating-visual-features","paper_date":"2024-06-14","arxiv_id":"2406.10082","code_links":[{"title":"roudimit/whisper-flamingo","url":"https://github.com/roudimit/whisper-flamingo"}],"syntology":{"n":18,"n_ran":11,"n_unverified":7,"n_pointer_only":18}}},{"leaderboard":"/sota/audio-visual-speech-recognition-on-lrw","slug":"audio-visual-speech-recognition-on-lrw","dataset":"LRW","dataset_url":"/dataset/lrw","rows_in_archive":3,"metrics":["Top-1 Accuracy"],"first_row_in_archive_order":{"model":"AVCRFormer","paper_title":"Audio-Visual Speech Recognition based on Regulated Transformer and Spatio-Temporal Fusion Strategy for Driver Assistive Systems","paper_url":"/paper/audio-visual-speech-recognition-based-on","paper_date":"2024-05-09","arxiv_id":null,"code_links":[{"title":"SMIL-SPCRAS/AVCRFormer","url":"https://github.com/SMIL-SPCRAS/AVCRFormer"}],"syntology":null}},{"leaderboard":"/sota/audio-visual-speech-recognition-on-cas-vsr","slug":"audio-visual-speech-recognition-on-cas-vsr","dataset":"CAS-VSR-S101","dataset_url":"/dataset/cas-vsr-s101","rows_in_archive":1,"metrics":["Word Error Rate (WER)"],"first_row_in_archive_order":{"model":"ES³ Base*","paper_title":"ES3: Evolving Self-Supervised Learning of Robust Audio-Visual Speech Representations","paper_url":"/paper/es3-evolving-self-supervised-learning-of","paper_date":"2024-01-01","arxiv_id":null,"code_links":[],"syntology":null}}],"datasets":[{"url":"/dataset/lrw","name":"LRW","full_name":"Lip Reading in the Wild","num_papers_in_archive":188},{"url":"/dataset/lrs2","name":"LRS2","full_name":"Lip Reading Sentences 2","num_papers_in_archive":115},{"url":"/dataset/how2","name":"How2","full_name":"","num_papers_in_archive":84},{"url":"/dataset/lrs3-ted","name":"LRS3-TED","full_name":"","num_papers_in_archive":63},{"url":"/dataset/lrw-1000","name":"CAS-VSR-W1k (LRW-1000)","full_name":"CAS-VSR-W1k (LRW-1000)","num_papers_in_archive":9},{"url":"/dataset/misp2021","name":"MISP2021","full_name":"Multimodal Information Based Speech Processing 2021","num_papers_in_archive":3},{"url":"/dataset/cas-vsr-s101","name":"CAS-VSR-S101","full_name":"","num_papers_in_archive":1}],"subtasks":[{"url":"/task/text-to-speech","name":"Text to Speech"}],"parent_tasks":[],"papers":{"order":"repositories listed in the archive (desc), then date (desc); the archive holds no stars","population":"papers tagged with this task that list at least one repository in the archive","shown":30,"of":42,"tagged_in_all":100,"items":[{"url":"/paper/deep-audio-visual-speech-recognition","title":"Deep Audio-Visual Speech Recognition","date":"2018-09-06","arxiv_id":"1809.02108","repositories_listed":4,"syntology":null},{"url":"/paper/end-to-end-audio-visual-speech-recognition","title":"End-to-end Audio-visual Speech Recognition with Conformers","date":"2021-02-12","arxiv_id":"2102.06657","repositories_listed":3,"syntology":null},{"url":"/paper/auto-avsr-audio-visual-speech-recognition","title":"Auto-AVSR: Audio-Visual Speech Recognition with Automatic Labels","date":"2023-03-25","arxiv_id":"2303.14307","repositories_listed":2,"syntology":{"n":6,"n_ran":3,"n_unverified":3,"n_pointer_only":0}},{"url":"/paper/discriminative-multi-modality-speech","title":"Discriminative Multi-modality Speech Recognition","date":"2020-05-12","arxiv_id":"2005.05592","repositories_listed":2,"syntology":null},{"url":"/paper/cogenav-versatile-audio-visual-representation","title":"CoGenAV: Versatile Audio-Visual Representation Learning via Contrastive-Generative Synchronization","date":"2025-05-06","arxiv_id":"2505.03186","repositories_listed":1,"syntology":null},{"url":"/paper/mms-llama-efficient-llm-based-audio-visual-1","title":"MMS-LLaMA: Efficient LLM-based Audio-Visual Speech Recognition with Minimal Multimodal Speech Tokens","date":"2025-03-14","arxiv_id":"2503.11315","repositories_listed":1,"syntology":null},{"url":"/paper/zero-avsr-zero-shot-audio-visual-speech","title":"Zero-AVSR: Zero-Shot Audio-Visual Speech Recognition with LLMs by Learning Language-Agnostic Speech Representations","date":"2025-03-08","arxiv_id":"2503.06273","repositories_listed":1,"syntology":null},{"url":"/paper/audio-visual-representation-learning-via","title":"Audio-Visual Representation Learning via Knowledge Distillation from Speech Foundation Models","date":"2025-02-09","arxiv_id":"2502.05766","repositories_listed":1,"syntology":null},{"url":"/paper/mwhisper-flamingo-for-multilingual-audio","title":"mWhisper-Flamingo for Multilingual Audio-Visual Noise-Robust Speech Recognition","date":"2025-02-03","arxiv_id":"2502.01547","repositories_listed":1,"syntology":null},{"url":"/paper/multi-task-corrupted-prediction-for-learning","title":"Multi-Task Corrupted Prediction for Learning Robust Audio-Visual Speech Representation","date":"2025-01-23","arxiv_id":"2504.18539","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_unverified":0,"n_pointer_only":0}},{"url":"/paper/listening-and-seeing-again-generative-error","title":"Listening and Seeing Again: Generative Error Correction for Audio-Visual Speech Recognition","date":"2025-01-03","arxiv_id":"2501.04038","repositories_listed":1,"syntology":null},{"url":"/paper/large-language-models-are-strong-audio-visual","title":"Large Language Models are Strong Audio-Visual Speech Recognition Learners","date":"2024-09-18","arxiv_id":"2409.12319","repositories_listed":1,"syntology":{"n":12,"n_ran":9,"n_unverified":3,"n_pointer_only":12}},{"url":"/paper/2408-00624","title":"SynesLM: A Unified Approach for Audio-visual Speech Recognition and Translation via Language Model and Synthetic Data","date":"2024-08-01","arxiv_id":"2408.00624","repositories_listed":1,"syntology":null},{"url":"/paper/tailored-design-of-audio-visual-speech","title":"Tailored Design of Audio-Visual Speech Recognition Models using Branchformers","date":"2024-07-09","arxiv_id":"2407.06606","repositories_listed":1,"syntology":null},{"url":"/paper/learning-video-temporal-dynamics-with-cross","title":"Learning Video Temporal Dynamics with Cross-Modal Attention for Robust Audio-Visual Speech Recognition","date":"2024-07-04","arxiv_id":"2407.03563","repositories_listed":1,"syntology":{"n":13,"n_ran":10,"n_unverified":3,"n_pointer_only":0}},{"url":"/paper/whisper-flamingo-integrating-visual-features","title":"Whisper-Flamingo: Integrating Visual Features into Whisper for Audio-Visual Speech Recognition and Translation","date":"2024-06-14","arxiv_id":"2406.10082","repositories_listed":1,"syntology":{"n":18,"n_ran":11,"n_unverified":7,"n_pointer_only":18}},{"url":"/paper/audio-visual-speech-recognition-based-on","title":"Audio-Visual Speech Recognition based on Regulated Transformer and Spatio-Temporal Fusion Strategy for Driver Assistive Systems","date":"2024-05-09","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/a-study-of-dropout-induced-modality-bias-on","title":"A Study of Dropout-Induced Modality Bias on Robustness to Missing Video Frames for Audio-Visual Speech Recognition","date":"2024-03-07","arxiv_id":"2403.04245","repositories_listed":1,"syntology":null},{"url":"/paper/it-s-never-too-late-fusing-acoustic","title":"It's Never Too Late: Fusing Acoustic Information into Large Language Models for Automatic Speech Recognition","date":"2024-02-08","arxiv_id":"2402.05457","repositories_listed":1,"syntology":{"n":6,"n_ran":3,"n_unverified":3,"n_pointer_only":0}},{"url":"/paper/multichannel-av-wav2vec2-a-framework-for","title":"Multichannel AV-wav2vec2: A Framework for Learning Multichannel Multi-Modal Speech Representation","date":"2024-01-07","arxiv_id":"2401.03468","repositories_listed":1,"syntology":{"n":9,"n_ran":6,"n_unverified":3,"n_pointer_only":9}},{"url":"/paper/rtfs-net-recurrent-time-frequency-modelling","title":"RTFS-Net: Recurrent Time-Frequency Modelling for Efficient Audio-Visual Speech Separation","date":"2023-09-29","arxiv_id":"2309.17189","repositories_listed":1,"syntology":{"n":1,"n_ran":0,"n_unverified":1,"n_pointer_only":0}},{"url":"/paper/improving-audio-visual-speech-recognition-by","title":"Improving Audio-Visual Speech Recognition by Lip-Subword Correlation Based Visual Pre-training and Cross-Modal Fusion Encoder","date":"2023-08-14","arxiv_id":"2308.08488","repositories_listed":1,"syntology":null},{"url":"/paper/mir-gan-refining-frame-level-modality","title":"MIR-GAN: Refining Frame-Level Modality-Invariant Representations with Adversarial Network for Audio-Visual Speech Recognition","date":"2023-06-18","arxiv_id":"2306.10567","repositories_listed":1,"syntology":{"n":1,"n_ran":1,"n_unverified":0,"n_pointer_only":1}},{"url":"/paper/hearing-lips-in-noise-universal-viseme","title":"Hearing Lips in Noise: Universal Viseme-Phoneme Mapping and Transfer for Robust Audio-Visual Speech Recognition","date":"2023-06-18","arxiv_id":"2306.10563","repositories_listed":1,"syntology":{"n":2,"n_ran":1,"n_unverified":1,"n_pointer_only":2}},{"url":"/paper/opensr-open-modality-speech-recognition-via","title":"OpenSR: Open-Modality Speech Recognition via Maintaining Multi-Modality Alignment","date":"2023-06-10","arxiv_id":"2306.06410","repositories_listed":1,"syntology":null},{"url":"/paper/mavd-the-first-open-large-scale-mandarin","title":"MAVD: The First Open Large-Scale Mandarin Audio-Visual Dataset with Depth Information","date":"2023-06-04","arxiv_id":"2306.02263","repositories_listed":1,"syntology":null},{"url":"/paper/prompting-the-hidden-talent-of-web-scale","title":"Prompting the Hidden Talent of Web-Scale Speech Models for Zero-Shot Task Generalization","date":"2023-05-18","arxiv_id":"2305.11095","repositories_listed":1,"syntology":null},{"url":"/paper/cross-modal-global-interaction-and-local","title":"Cross-Modal Global Interaction and Local Alignment for Audio-Visual Speech Recognition","date":"2023-05-16","arxiv_id":"2305.09212","repositories_listed":1,"syntology":null},{"url":"/paper/watch-or-listen-robust-audio-visual-speech","title":"Watch or Listen: Robust Audio-Visual Speech Recognition with Visual Corruption Modeling and Reliability Scoring","date":"2023-03-15","arxiv_id":"2303.08536","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_unverified":0,"n_pointer_only":2}},{"url":"/paper/muavic-a-multilingual-audio-visual-corpus-for","title":"MuAViC: A Multilingual Audio-Visual Corpus for Robust Speech Recognition and Robust Speech-to-Text Translation","date":"2023-03-01","arxiv_id":"2303.00628","repositories_listed":1,"syntology":null}],"syntology_records":11,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-09-25T09:33:49+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"}},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"}}