{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/unified-speech-recognition-a-single-model-for","title":"Unified Speech Recognition: A Single Model for Auditory, Visual, and Audiovisual Inputs","arxiv_id":"2411.02256","date":"2024-11-04","proceeding":null,"authors":["Alexandros Haliassos","Rodrigo Mira","Honglie Chen","Zoe Landgraf","Stavros Petridis","Maja Pantic"],"abstract":"Research in auditory, visual, and audiovisual speech recognition (ASR, VSR, and AVSR, respectively) has traditionally been conducted independently. Even recent self-supervised studies addressing two or all three tasks simultaneously tend to yield separate models, leading to disjoint inference pipelines with increased memory requirements and redundancies. This paper proposes unified training strategies for these systems. We demonstrate that training a single model for all three tasks enhances VSR and AVSR performance, overcoming typical optimisation challenges when training from scratch. Moreover, we introduce a greedy pseudo-labelling approach to more effectively leverage unlabelled samples, addressing shortcomings in related self-supervised methods. Finally, we develop a self-supervised pre-training method within our framework, proving its effectiveness alongside our semi-supervised approach. Despite using a single model for all tasks, our unified approach achieves state-of-the-art performance compared to recent methods on LRS3 and LRS2 for ASR, VSR, and AVSR, as well as on the newly released WildVSR dataset. Code and models are available at https://github.com/ahaliassos/usr.","url_abs":"https://arxiv.org/abs/2411.02256v1","url_pdf":"https://arxiv.org/pdf/2411.02256v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"unified-speech-recognition-a-single-model-for","repo_url":"https://github.com/ahaliassos/usr","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"lipreading","task_name":"Lipreading"},{"task_slug":"speech-recognition","task_name":"Speech Recognition"},{"task_slug":"speech-recognition-1","task_name":"speech-recognition"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/lipreading-on-lrs2","task":"Lipreading","dataset":"LRS2","model":"USR","rank_in_archive_order":2,"of":25,"metrics":{"Word Error Rate (WER)":"15.4"},"uses_additional_data":true},{"leaderboard":"/sota/lipreading-on-lrs3-ted","task":"Lipreading","dataset":"LRS3-TED","model":"USR (self + semi-supervised)","rank_in_archive_order":4,"of":23,"metrics":{"Word Error Rate (WER)":"21.5"},"uses_additional_data":true},{"leaderboard":"/sota/lipreading-on-lrs3-ted","task":"Lipreading","dataset":"LRS3-TED","model":"USR (self-supervised)","rank_in_archive_order":5,"of":23,"metrics":{"Word Error Rate (WER)":"22.3"},"uses_additional_data":true}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2411.02256","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2411.02256"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ahaliassos/usr","reach":{"status":"ok"}}],"summary":{"unverified":9},"by_repo_kind":{"official":{"samples":9,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":9,"samples":[{"code_sha256_prefix":"bb3665ddb6b8dd2d","entry":"adadelta_eps_decay","repo":"ahaliassos/usr","repo_kind":"official","path":"espnet/asr/asr_utils.py","file_url":"https://github.com/ahaliassos/usr/blob/HEAD/espnet/asr/asr_utils.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bb3665ddb6b8dd2d"}},{"code_sha256_prefix":"1fa5ca494ed9750a","entry":"adam_lr_decay","repo":"ahaliassos/usr","repo_kind":"official","path":"espnet/asr/asr_utils.py","file_url":"https://github.com/ahaliassos/usr/blob/HEAD/espnet/asr/asr_utils.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1fa5ca494ed9750a"}},{"code_sha256_prefix":"c844dae8bf36cb0d","entry":"batch_shuffle_ddp","repo":"ahaliassos/usr","repo_kind":"official","path":"models/utils.py","file_url":"https://github.com/ahaliassos/usr/blob/HEAD/models/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c844dae8bf36cb0d"}},{"code_sha256_prefix":"1ac98ef4b97d4dfc","entry":"batch_unshuffle_ddp","repo":"ahaliassos/usr","repo_kind":"official","path":"models/utils.py","file_url":"https://github.com/ahaliassos/usr/blob/HEAD/models/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1ac98ef4b97d4dfc"}},{"code_sha256_prefix":"63d6be75146d60ff","entry":"concat_all_gather","repo":"ahaliassos/usr","repo_kind":"official","path":"models/utils.py","file_url":"https://github.com/ahaliassos/usr/blob/HEAD/models/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"63d6be75146d60ff"}},{"code_sha256_prefix":"c97dab612dfcc0bd","entry":"get_cer","repo":"ahaliassos/usr","repo_kind":"official","path":"metrics.py","file_url":"https://github.com/ahaliassos/usr/blob/HEAD/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c97dab612dfcc0bd"}},{"code_sha256_prefix":"f646df9b796b4337","entry":"get_er","repo":"ahaliassos/usr","repo_kind":"official","path":"metrics.py","file_url":"https://github.com/ahaliassos/usr/blob/HEAD/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f646df9b796b4337"}},{"code_sha256_prefix":"f0ad236b3b39fdaf","entry":"get_wer","repo":"ahaliassos/usr","repo_kind":"official","path":"metrics.py","file_url":"https://github.com/ahaliassos/usr/blob/HEAD/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f0ad236b3b39fdaf"}},{"code_sha256_prefix":"96081a60369c59e0","entry":"restore_snapshot","repo":"ahaliassos/usr","repo_kind":"official","path":"espnet/asr/asr_utils.py","file_url":"https://github.com/ahaliassos/usr/blob/HEAD/espnet/asr/asr_utils.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"96081a60369c59e0"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}