{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/lipnet-end-to-end-sentence-level-lipreading","title":"LipNet: End-to-End Sentence-level Lipreading","arxiv_id":"1611.01599","date":"2016-11-05","proceeding":null,"authors":["Yannis M. Assael","Brendan Shillingford","Shimon Whiteson","Nando de Freitas"],"abstract":"Lipreading is the task of decoding text from the movement of a speaker's\nmouth. Traditional approaches separated the problem into two stages: designing\nor learning visual features, and prediction. More recent deep lipreading\napproaches are end-to-end trainable (Wand et al., 2016; Chung & Zisserman,\n2016a). However, existing work on models trained end-to-end perform only word\nclassification, rather than sentence-level sequence prediction. Studies have\nshown that human lipreading performance increases for longer words (Easton &\nBasala, 1982), indicating the importance of features capturing temporal context\nin an ambiguous communication channel. Motivated by this observation, we\npresent LipNet, a model that maps a variable-length sequence of video frames to\ntext, making use of spatiotemporal convolutions, a recurrent network, and the\nconnectionist temporal classification loss, trained entirely end-to-end. To the\nbest of our knowledge, LipNet is the first end-to-end sentence-level lipreading\nmodel that simultaneously learns spatiotemporal visual features and a sequence\nmodel. On the GRID corpus, LipNet achieves 95.2% accuracy in sentence-level,\noverlapped speaker split task, outperforming experienced human lipreaders and\nthe previous 86.4% word-level state-of-the-art accuracy (Gergen et al., 2016).","url_abs":"http://arxiv.org/abs/1611.01599v2","url_pdf":"http://arxiv.org/pdf/1611.01599v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"lipnet-end-to-end-sentence-level-lipreading","repo_url":"https://github.com/rizkiarm/LipNet","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"lipnet-end-to-end-sentence-level-lipreading","repo_url":"https://github.com/Abishalini/LipReadingGUI","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"lipnet-end-to-end-sentence-level-lipreading","repo_url":"https://github.com/Fengdalu/LipNet-PyTorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"lipnet-end-to-end-sentence-level-lipreading","repo_url":"https://github.com/LiZhenghua0311/lip","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"lipnet-end-to-end-sentence-level-lipreading","repo_url":"https://github.com/PlatDrake2875/LipNet","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"lipnet-end-to-end-sentence-level-lipreading","repo_url":"https://github.com/SohaibAnwaar/lip-Reading-by-Deep-learning","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"lipnet-end-to-end-sentence-level-lipreading","repo_url":"https://github.com/hero9968/lipnet-python","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"lipnet-end-to-end-sentence-level-lipreading","repo_url":"https://github.com/ms8909/LipONet","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"lipnet-end-to-end-sentence-level-lipreading","repo_url":"https://github.com/pjenpoomjai/LipNet","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"lipnet-end-to-end-sentence-level-lipreading","repo_url":"https://github.com/sailordiary/LipNet-PyTorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"BSD-3-Clause"}},{"paper_slug":"lipnet-end-to-end-sentence-level-lipreading","repo_url":"https://github.com/ski-net/lipnet","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"mxnet","reach":{"status":"ok"}},{"paper_slug":"lipnet-end-to-end-sentence-level-lipreading","repo_url":"https://github.com/speech-separation-hse/video-features","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"lipnet-end-to-end-sentence-level-lipreading","repo_url":"https://github.com/PatrickPrakash/LiptoSpeech","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"tf","reach":{"status":"ok"}}],"tasks":[{"task_slug":"classification","task_name":"General Classification"},{"task_slug":"lipreading","task_name":"Lipreading"},{"task_slug":"sentence","task_name":"Sentence"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/lipreading-on-grid-corpus-mixed-speech","task":"Lipreading","dataset":"GRID corpus (mixed-speech)","model":"LipNet","rank_in_archive_order":5,"of":5,"metrics":{"Word Error Rate (WER)":"4.6"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/1611.01599","atlas_url":"https://app.syntology.ai/?focus=1611.01599","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1611.01599"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/PatrickPrakash/LiptoSpeech","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/pjenpoomjai/LipNet","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/speech-separation-hse/video-features","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sailordiary/LipNet-PyTorch","reach":{"status":"ok","spdx":"BSD-3-Clause"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ski-net/lipnet","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Abishalini/LipReadingGUI","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Fengdalu/LipNet-PyTorch","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/hero9968/lipnet-python","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ms8909/LipONet","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/rizkiarm/LipNet","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/LiZhenghua0311/lip","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/SohaibAnwaar/lip-Reading-by-Deep-learning","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/PlatDrake2875/LipNet","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":15},"by_repo_kind":{"official":{"samples":5,"ran":0,"repositories":1},"listed":{"samples":10,"ran":0,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"88b8acae6777424e","entry":"ctc_collate","repo":"sailordiary/LipNet-PyTorch","repo_kind":"listed","path":"dataloader.py","file_url":"https://github.com/sailordiary/LipNet-PyTorch/blob/HEAD/dataloader.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"88b8acae6777424e"}},{"code_sha256_prefix":"8181d94a6381a30a","entry":"ctc_lambda_func","repo":"rizkiarm/LipNet","repo_kind":"official","path":"lipnet/core/loss.py","file_url":"https://github.com/rizkiarm/LipNet/blob/HEAD/lipnet/core/loss.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8181d94a6381a30a"}},{"code_sha256_prefix":"07d26c38ea615861","entry":"decode","repo":"rizkiarm/LipNet","repo_kind":"official","path":"lipnet/core/decoders.py","file_url":"https://github.com/rizkiarm/LipNet/blob/HEAD/lipnet/core/decoders.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"07d26c38ea615861"}},{"code_sha256_prefix":"bc11ac6762ddee1c","entry":"get_file_extension","repo":"pjenpoomjai/LipNet","repo_kind":"listed","path":"common/files.py","file_url":"https://github.com/pjenpoomjai/LipNet/blob/HEAD/common/files.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bc11ac6762ddee1c"}},{"code_sha256_prefix":"0ab02e700216530c","entry":"is_dir","repo":"pjenpoomjai/LipNet","repo_kind":"listed","path":"common/files.py","file_url":"https://github.com/pjenpoomjai/LipNet/blob/HEAD/common/files.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0ab02e700216530c"}},{"code_sha256_prefix":"74ffcb74cead6878","entry":"is_file","repo":"pjenpoomjai/LipNet","repo_kind":"listed","path":"common/files.py","file_url":"https://github.com/pjenpoomjai/LipNet/blob/HEAD/common/files.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"74ffcb74cead6878"}},{"code_sha256_prefix":"ba5f0f129eba0ac3","entry":"labels_to_text","repo":"rizkiarm/LipNet","repo_kind":"official","path":"lipnet/lipreading/helpers.py","file_url":"https://github.com/rizkiarm/LipNet/blob/HEAD/lipnet/lipreading/helpers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ba5f0f129eba0ac3"}},{"code_sha256_prefix":"f3ec786915134721","entry":"read_data","repo":"sailordiary/LipNet-PyTorch","repo_kind":"listed","path":"augmentation.py","file_url":"https://github.com/sailordiary/LipNet-PyTorch/blob/HEAD/augmentation.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"f3ec786915134721"}},{"code_sha256_prefix":"6460635662358375","entry":"round","repo":"sailordiary/LipNet-PyTorch","repo_kind":"listed","path":"augmentation.py","file_url":"https://github.com/sailordiary/LipNet-PyTorch/blob/HEAD/augmentation.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"6460635662358375"}},{"code_sha256_prefix":"80b5790c95657203","entry":"round","repo":"sailordiary/LipNet-PyTorch","repo_kind":"listed","path":"dataloader.py","file_url":"https://github.com/sailordiary/LipNet-PyTorch/blob/HEAD/dataloader.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"80b5790c95657203"}},{"code_sha256_prefix":"071ed9f14cb9d384","entry":"text_to_labels","repo":"rizkiarm/LipNet","repo_kind":"official","path":"lipnet/lipreading/helpers.py","file_url":"https://github.com/rizkiarm/LipNet/blob/HEAD/lipnet/lipreading/helpers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"071ed9f14cb9d384"}},{"code_sha256_prefix":"6656877b73807105","entry":"threadsafe_generator","repo":"rizkiarm/LipNet","repo_kind":"official","path":"lipnet/helpers/threadsafe.py","file_url":"https://github.com/rizkiarm/LipNet/blob/HEAD/lipnet/helpers/threadsafe.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6656877b73807105"}},{"code_sha256_prefix":"9108c44188636656","entry":"tokenize","repo":"pjenpoomjai/LipNet","repo_kind":"listed","path":"core/decoding/spell.py","file_url":"https://github.com/pjenpoomjai/LipNet/blob/HEAD/core/decoding/spell.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9108c44188636656"}},{"code_sha256_prefix":"8b832c2b8bf29a51","entry":"untokenize","repo":"pjenpoomjai/LipNet","repo_kind":"listed","path":"core/decoding/spell.py","file_url":"https://github.com/pjenpoomjai/LipNet/blob/HEAD/core/decoding/spell.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8b832c2b8bf29a51"}},{"code_sha256_prefix":"face0a3bb9c1c095","entry":"video_to_frames","repo":"pjenpoomjai/LipNet","repo_kind":"listed","path":"core/video.py","file_url":"https://github.com/pjenpoomjai/LipNet/blob/HEAD/core/video.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"face0a3bb9c1c095"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}