{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/learning-video-temporal-dynamics-with-cross","title":"Learning Video Temporal Dynamics with Cross-Modal Attention for Robust Audio-Visual Speech Recognition","arxiv_id":"2407.03563","date":"2024-07-04","proceeding":null,"authors":["Sungnyun Kim","Kangwook Jang","Sangmin Bae","Hoirin Kim","Se-Young Yun"],"abstract":"Audio-visual speech recognition (AVSR) aims to transcribe human speech using both audio and video modalities. In practical environments with noise-corrupted audio, the role of video information becomes crucial. However, prior works have primarily focused on enhancing audio features in AVSR, overlooking the importance of video features. In this study, we strengthen the video features by learning three temporal dynamics in video data: context order, playback direction, and the speed of video frames. Cross-modal attention modules are introduced to enrich video features with audio information so that speech variability can be taken into account when training on the video temporal dynamics. Based on our approach, we achieve the state-of-the-art performance on the LRS2 and LRS3 AVSR benchmarks for the noise-dominant settings. Our approach excels in scenarios especially for babble and speech noise, indicating the ability to distinguish the speech signal that should be recognized from lip movements in the video modality. We support the validity of our methodology by offering the ablation experiments for the temporal dynamics losses and the cross-modal attention architecture design.","url_abs":"https://arxiv.org/abs/2407.03563v3","url_pdf":"https://arxiv.org/pdf/2407.03563v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"learning-video-temporal-dynamics-with-cross","repo_url":"https://github.com/sungnyun/avsr-temporal-dynamics","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"audio-visual-speech-recognition","task_name":"Audio-Visual Speech Recognition"},{"task_slug":"speech-recognition","task_name":"Speech Recognition"},{"task_slug":"visual-speech-recognition","task_name":"Visual Speech Recognition"},{"task_slug":"speech-recognition-1","task_name":"speech-recognition"}],"methods":[{"method_slug":"attention","method_name":"Attention"},{"method_slug":"speed","method_name":"SPEED"},{"method_slug":"softmax","method_name":"Softmax"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2407.03563","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.03563"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sungnyun/avsr-temporal-dynamics","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_draft_wrong":4,"ran_honours":1,"ran":5,"unverified":3},"by_repo_kind":{"official":{"samples":13,"ran":10,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"96cbb5e9ca5b6be0","entry":"Embedding","repo":"sungnyun/avsr-temporal-dynamics","repo_kind":"official","path":"avhubert/hubert_asr.py","file_url":"https://github.com/sungnyun/avsr-temporal-dynamics/blob/HEAD/avhubert/hubert_asr.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"96cbb5e9ca5b6be0"}},{"code_sha256_prefix":"8cd8cb0d1e9e63e4","entry":"Linear","repo":"sungnyun/avsr-temporal-dynamics","repo_kind":"official","path":"avhubert/hubert_asr.py","file_url":"https://github.com/sungnyun/avsr-temporal-dynamics/blob/HEAD/avhubert/hubert_asr.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8cd8cb0d1e9e63e4"}},{"code_sha256_prefix":"357d3a4f0be679d9","entry":"apply_half","repo":"sungnyun/avsr-temporal-dynamics","repo_kind":"official","path":"avhubert/infer_s2s.py","file_url":"https://github.com/sungnyun/avsr-temporal-dynamics/blob/HEAD/avhubert/infer_s2s.py","link_basis":"plan_row","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"357d3a4f0be679d9"}},{"code_sha256_prefix":"581de97f5d5aa97a","entry":"compute_mask_indices","repo":"sungnyun/avsr-temporal-dynamics","repo_kind":"official","path":"avhubert/utils.py","file_url":"https://github.com/sungnyun/avsr-temporal-dynamics/blob/HEAD/avhubert/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"581de97f5d5aa97a"}},{"code_sha256_prefix":"583f9780bdd00a45","entry":"conv3x3","repo":"sungnyun/avsr-temporal-dynamics","repo_kind":"official","path":"avhubert/resnet.py","file_url":"https://github.com/sungnyun/avsr-temporal-dynamics/blob/HEAD/avhubert/resnet.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"583f9780bdd00a45"}},{"code_sha256_prefix":"d50e3a8f0b0f9a21","entry":"downsample_basic_block","repo":"sungnyun/avsr-temporal-dynamics","repo_kind":"official","path":"avhubert/resnet.py","file_url":"https://github.com/sungnyun/avsr-temporal-dynamics/blob/HEAD/avhubert/resnet.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d50e3a8f0b0f9a21"}},{"code_sha256_prefix":"546a770034e200d1","entry":"downsample_basic_block_v2","repo":"sungnyun/avsr-temporal-dynamics","repo_kind":"official","path":"avhubert/resnet.py","file_url":"https://github.com/sungnyun/avsr-temporal-dynamics/blob/HEAD/avhubert/resnet.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"546a770034e200d1"}},{"code_sha256_prefix":"db4a33e22cd9f644","entry":"find_runs","repo":"sungnyun/avsr-temporal-dynamics","repo_kind":"official","path":"avhubert/utils.py","file_url":"https://github.com/sungnyun/avsr-temporal-dynamics/blob/HEAD/avhubert/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"db4a33e22cd9f644"}},{"code_sha256_prefix":"bbe1a698e337d4d4","entry":"get_symbols_to_strip_from_output","repo":"sungnyun/avsr-temporal-dynamics","repo_kind":"official","path":"avhubert/infer_s2s.py","file_url":"https://github.com/sungnyun/avsr-temporal-dynamics/blob/HEAD/avhubert/infer_s2s.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bbe1a698e337d4d4"}},{"code_sha256_prefix":"d5a7afb95212031e","entry":"load_video","repo":"sungnyun/avsr-temporal-dynamics","repo_kind":"official","path":"avhubert/utils.py","file_url":"https://github.com/sungnyun/avsr-temporal-dynamics/blob/HEAD/avhubert/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d5a7afb95212031e"}},{"code_sha256_prefix":"f0641584d0fe0fb5","entry":"load_audio_visual","repo":"sungnyun/avsr-temporal-dynamics","repo_kind":"official","path":"avhubert/hubert_dataset.py","file_url":"https://github.com/sungnyun/avsr-temporal-dynamics/blob/HEAD/avhubert/hubert_dataset.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f0641584d0fe0fb5"}},{"code_sha256_prefix":"113a523619225ba0","entry":"load_label","repo":"sungnyun/avsr-temporal-dynamics","repo_kind":"official","path":"avhubert/hubert_dataset.py","file_url":"https://github.com/sungnyun/avsr-temporal-dynamics/blob/HEAD/avhubert/hubert_dataset.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"113a523619225ba0"}},{"code_sha256_prefix":"3e9450b6ac2fcddd","entry":"load_label_offset","repo":"sungnyun/avsr-temporal-dynamics","repo_kind":"official","path":"avhubert/hubert_dataset.py","file_url":"https://github.com/sungnyun/avsr-temporal-dynamics/blob/HEAD/avhubert/hubert_dataset.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3e9450b6ac2fcddd"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}