{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/tim-a-time-interval-machine-for-audio-visual","title":"TIM: A Time Interval Machine for Audio-Visual Action Recognition","arxiv_id":"2404.05559","date":"2024-04-08","proceeding":"CVPR 2024 1","authors":["Jacob Chalk","Jaesung Huh","Evangelos Kazakos","Andrew Zisserman","Dima Damen"],"abstract":"Diverse actions give rise to rich audio-visual signals in long videos. Recent works showcase that the two modalities of audio and video exhibit different temporal extents of events and distinct labels. We address the interplay between the two modalities in long videos by explicitly modelling the temporal extents of audio and visual events. We propose the Time Interval Machine (TIM) where a modality-specific time interval poses as a query to a transformer encoder that ingests a long video input. The encoder then attends to the specified interval, as well as the surrounding context in both modalities, in order to recognise the ongoing action. We test TIM on three long audio-visual video datasets: EPIC-KITCHENS, Perception Test, and AVE, reporting state-of-the-art (SOTA) for recognition. On EPIC-KITCHENS, we beat previous SOTA that utilises LLMs and significantly larger pre-training by 2.9% top-1 action recognition accuracy. Additionally, we show that TIM can be adapted for action detection, using dense multi-scale interval queries, outperforming SOTA on EPIC-KITCHENS-100 for most metrics, and showing strong performance on the Perception Test. Our ablations show the critical role of integrating the two modalities and modelling their time intervals in achieving this performance. Code and models at: https://github.com/JacobChalk/TIM","url_abs":"https://arxiv.org/abs/2404.05559v2","url_pdf":"https://arxiv.org/pdf/2404.05559v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"tim-a-time-interval-machine-for-audio-visual","repo_url":"https://github.com/jacobchalk/tim","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"action-detection","task_name":"Action Detection"},{"task_slug":"action-recognition-in-videos","task_name":"Action Recognition"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/action-recognition-on-epic-kitchens-100","task":"Action Recognition","dataset":"EPIC-KITCHENS-100","model":"TIM","rank_in_archive_order":2,"of":32,"metrics":{"Action@1":"56.4","Noun@1":"66.4","Verb@1":"76.2"},"uses_additional_data":true}],"syntology":{"syntology_url":"https://syntology.ai/paper/2404.05559","atlas_url":"https://app.syntology.ai/?focus=2404.05559","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.05559"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jacobchalk/tim","reach":{"status":"ok"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/JacobChalk/TIM","reach":{"status":"ok"}}],"summary":{"ran_honours":1,"ran":10,"unverified":1},"by_repo_kind":{"official":{"samples":12,"ran":11,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":12,"samples":[{"code_sha256_prefix":"da651e3979a18f84","entry":"get_sinusoid_encoding_table","repo":"JacobChalk/TIM","repo_kind":"official","path":"feature_extractors/VideoMAE/modeling_finetune.py","file_url":"https://github.com/JacobChalk/TIM/blob/HEAD/feature_extractors/VideoMAE/modeling_finetune.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"da651e3979a18f84"}},{"code_sha256_prefix":"3777cf3d1476af2d","entry":"load_gt_segmentations","repo":"JacobChalk/TIM","repo_kind":"official","path":"detection/eval_detection/evaluate_detection_json.py","file_url":"https://github.com/JacobChalk/TIM/blob/HEAD/detection/eval_detection/evaluate_detection_json.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3777cf3d1476af2d"}},{"code_sha256_prefix":"5a45614aa6767966","entry":"load_gt_segmentations","repo":"JacobChalk/TIM","repo_kind":"official","path":"detection/eval_detection/evaluate_detection_json_ek100.py","file_url":"https://github.com/JacobChalk/TIM/blob/HEAD/detection/eval_detection/evaluate_detection_json_ek100.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5a45614aa6767966"}},{"code_sha256_prefix":"91dbf25ea34df77b","entry":"load_predicted_segmentations","repo":"JacobChalk/TIM","repo_kind":"official","path":"detection/eval_detection/evaluate_detection_json.py","file_url":"https://github.com/JacobChalk/TIM/blob/HEAD/detection/eval_detection/evaluate_detection_json.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"91dbf25ea34df77b"}},{"code_sha256_prefix":"67d66192df8ebf14","entry":"load_predicted_segmentations","repo":"JacobChalk/TIM","repo_kind":"official","path":"detection/eval_detection/evaluate_detection_json_ek100.py","file_url":"https://github.com/JacobChalk/TIM/blob/HEAD/detection/eval_detection/evaluate_detection_json_ek100.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"67d66192df8ebf14"}},{"code_sha256_prefix":"eb3884a04aa0a2ed","entry":"sec_to_timestamp","repo":"JacobChalk/TIM","repo_kind":"official","path":"feature_extractors/make_framepickle.py","file_url":"https://github.com/JacobChalk/TIM/blob/HEAD/feature_extractors/make_framepickle.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"eb3884a04aa0a2ed"}},{"code_sha256_prefix":"2e4e22e309ff1d23","entry":"sec_to_timestamp","repo":"JacobChalk/TIM","repo_kind":"official","path":"feature_extractors/make_videoinfo.py","file_url":"https://github.com/JacobChalk/TIM/blob/HEAD/feature_extractors/make_videoinfo.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2e4e22e309ff1d23"}},{"code_sha256_prefix":"8a8e3d27500e4fd8","entry":"timestamp_to_sec","repo":"JacobChalk/TIM","repo_kind":"official","path":"feature_extractors/make_framepickle.py","file_url":"https://github.com/JacobChalk/TIM/blob/HEAD/feature_extractors/make_framepickle.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8a8e3d27500e4fd8"}},{"code_sha256_prefix":"61dfbd1801e160f7","entry":"timestamp_to_seconds","repo":"JacobChalk/TIM","repo_kind":"official","path":"feature_extractors/make_videoinfo.py","file_url":"https://github.com/JacobChalk/TIM/blob/HEAD/feature_extractors/make_videoinfo.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"61dfbd1801e160f7"}},{"code_sha256_prefix":"d45a7468dd11e23d","entry":"timestamp_to_seconds","repo":"JacobChalk/TIM","repo_kind":"official","path":"detection/eval_detection/evaluate_detection_json.py","file_url":"https://github.com/JacobChalk/TIM/blob/HEAD/detection/eval_detection/evaluate_detection_json.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d45a7468dd11e23d"}},{"code_sha256_prefix":"93f25c853b7eeac5","entry":"train_class_batch","repo":"JacobChalk/TIM","repo_kind":"official","path":"feature_extractors/VideoMAE/engine_for_finetuning.py","file_url":"https://github.com/JacobChalk/TIM/blob/HEAD/feature_extractors/VideoMAE/engine_for_finetuning.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"93f25c853b7eeac5"}},{"code_sha256_prefix":"29416043c7035e7c","entry":"get_loss_scale_for_deepspeed","repo":"JacobChalk/TIM","repo_kind":"official","path":"feature_extractors/VideoMAE/engine_for_finetuning.py","file_url":"https://github.com/JacobChalk/TIM/blob/HEAD/feature_extractors/VideoMAE/engine_for_finetuning.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"29416043c7035e7c"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}