{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/implicit-temporal-modeling-with-learnable","title":"Implicit Temporal Modeling with Learnable Alignment for Video Recognition","arxiv_id":"2304.10465","date":"2023-04-20","proceeding":"ICCV 2023 1","authors":["Shuyuan Tu","Qi Dai","Zuxuan Wu","Zhi-Qi Cheng","Han Hu","Yu-Gang Jiang"],"abstract":"Contrastive language-image pretraining (CLIP) has demonstrated remarkable success in various image tasks. However, how to extend CLIP with effective temporal modeling is still an open and crucial problem. Existing factorized or joint spatial-temporal modeling trades off between the efficiency and performance. While modeling temporal information within straight through tube is widely adopted in literature, we find that simple frame alignment already provides enough essence without temporal attention. To this end, in this paper, we proposed a novel Implicit Learnable Alignment (ILA) method, which minimizes the temporal modeling effort while achieving incredibly high performance. Specifically, for a frame pair, an interactive point is predicted in each frame, serving as a mutual information rich region. By enhancing the features around the interactive point, two frames are implicitly aligned. The aligned features are then pooled into a single token, which is leveraged in the subsequent spatial self-attention. Our method allows eliminating the costly or insufficient temporal self-attention in video. Extensive experiments on benchmarks demonstrate the superiority and generality of our module. Particularly, the proposed ILA achieves a top-1 accuracy of 88.7% on Kinetics-400 with much fewer FLOPs compared with Swin-L and ViViT-H. Code is released at https://github.com/Francis-Rings/ILA .","url_abs":"https://arxiv.org/abs/2304.10465v2","url_pdf":"https://arxiv.org/pdf/2304.10465v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"implicit-temporal-modeling-with-learnable","repo_url":"https://github.com/francis-rings/ila","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"action-classification","task_name":"Action Classification"},{"task_slug":"action-recognition-in-videos","task_name":"Action Recognition"},{"task_slug":"video-recognition","task_name":"Video Recognition"}],"methods":[{"method_slug":"clip","method_name":"CLIP"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/action-classification-on-kinetics-400","task":"Action Classification","dataset":"Kinetics-400","model":"ILA (ViT-L/14)","rank_in_archive_order":22,"of":207,"metrics":{"Acc@1":"88.7","Acc@5":"97.8"},"uses_additional_data":false},{"leaderboard":"/sota/action-classification-on-kinetics-400","task":"Action Classification","dataset":"Kinetics-400","model":"ILA (ViT-B/16)","rank_in_archive_order":52,"of":207,"metrics":{"Acc@1":"85.7","Acc@5":"97.2"},"uses_additional_data":false},{"leaderboard":"/sota/action-classification-on-kinetics-400","task":"Action Classification","dataset":"Kinetics-400","model":"ILA (ViT-B/32)","rank_in_archive_order":78,"of":207,"metrics":{"Acc@1":"82.4","Acc@5":"95.8"},"uses_additional_data":false},{"leaderboard":"/sota/action-recognition-in-videos-on-something","task":"Action Recognition","dataset":"Something-Something V2","model":"ILA (ViT-L/14)","rank_in_archive_order":38,"of":123,"metrics":{"Top-1 Accuracy":"70.2","Top-5 Accuracy":"91.8"},"uses_additional_data":false},{"leaderboard":"/sota/action-recognition-in-videos-on-something","task":"Action Recognition","dataset":"Something-Something V2","model":"ILA (ViT-B/16)","rank_in_archive_order":75,"of":123,"metrics":{"Top-1 Accuracy":"66.8","Top-5 Accuracy":"90.3"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2304.10465","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2304.10465"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/francis-rings/ila","reach":{"status":"ok"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/Francis-Rings/ILA","reach":{"status":"ok"}}],"summary":{"ran":1,"ran_draft_wrong":1,"unverified":2},"by_repo_kind":{"official":{"samples":4,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":4,"samples":[{"code_sha256_prefix":"11d331bace01b706","entry":"Depth_Separable_Convolution","repo":"Francis-Rings/ILA","repo_kind":"official","path":"models/align.py","file_url":"https://github.com/Francis-Rings/ILA/blob/HEAD/models/align.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"11d331bace01b706"}},{"code_sha256_prefix":"cf55039afb930528","entry":"aligned_mask_generation","repo":"Francis-Rings/ILA","repo_kind":"official","path":"models/align.py","file_url":"https://github.com/Francis-Rings/ILA/blob/HEAD/models/align.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"cf55039afb930528"}},{"code_sha256_prefix":"e40c02c37eaa40de","entry":"ILA","repo":"Francis-Rings/ILA","repo_kind":"official","path":"models/align.py","file_url":"https://github.com/Francis-Rings/ILA/blob/HEAD/models/align.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e40c02c37eaa40de"}},{"code_sha256_prefix":"694fd30d82fcd358","entry":"SqueezeAndRerange","repo":"Francis-Rings/ILA","repo_kind":"official","path":"models/align.py","file_url":"https://github.com/Francis-Rings/ILA/blob/HEAD/models/align.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"694fd30d82fcd358"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}