{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/frozen-in-time-a-joint-video-and-image","title":"Frozen in Time: A Joint Video and Image Encoder for End-to-End Retrieval","arxiv_id":"2104.00650","date":"2021-04-01","proceeding":"ICCV 2021 10","authors":["Max Bain","Arsha Nagrani","Gül Varol","Andrew Zisserman"],"abstract":"Our objective in this work is video-text retrieval - in particular a joint embedding that enables efficient text-to-video retrieval. The challenges in this area include the design of the visual architecture and the nature of the training data, in that the available large scale video-text training datasets, such as HowTo100M, are noisy and hence competitive performance is achieved only at scale through large amounts of compute. We address both these challenges in this paper. We propose an end-to-end trainable model that is designed to take advantage of both large-scale image and video captioning datasets. Our model is an adaptation and extension of the recent ViT and Timesformer architectures, and consists of attention in both space and time. The model is flexible and can be trained on both image and video text datasets, either independently or in conjunction. It is trained with a curriculum learning schedule that begins by treating images as 'frozen' snapshots of video, and then gradually learns to attend to increasing temporal context when trained on video datasets. We also provide a new video-text pretraining dataset WebVid-2M, comprised of over two million videos with weak captions scraped from the internet. Despite training on datasets that are an order of magnitude smaller, we show that this approach yields state-of-the-art results on standard downstream video-retrieval benchmarks including MSR-VTT, MSVD, DiDeMo and LSMDC.","url_abs":"https://arxiv.org/abs/2104.00650v2","url_pdf":"https://arxiv.org/pdf/2104.00650v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"frozen-in-time-a-joint-video-and-image","repo_url":"https://github.com/m-bain/frozen-in-time","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"frozen-in-time-a-joint-video-and-image","repo_url":"https://github.com/m-bain/webvid","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"frozen-in-time-a-joint-video-and-image","repo_url":"https://github.com/princetonvisualai/mqvr","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"frozen-in-time-a-joint-video-and-image","repo_url":"https://github.com/willard-yuan/video-text-retrieval-papers","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"frozen-in-time-a-joint-video-and-image","repo_url":"https://github.com/towhee-io/towhee","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"text-retrieval","task_name":"Text Retrieval"},{"task_slug":"text-to-video-retrieval","task_name":"Text to Video Retrieval"},{"task_slug":"video-captioning","task_name":"Video Captioning"},{"task_slug":"video-retrieval","task_name":"Video Retrieval"},{"task_slug":"video-text-retrieval","task_name":"Video-Text Retrieval"},{"task_slug":"zero-shot-video-retrieval","task_name":"Zero-Shot Video Retrieval"}],"methods":[],"datasets_introduced":[{"slug":"webvid","name":"WebVid","full_name":""}],"methods_introduced":[],"results":[{"leaderboard":"/sota/video-retrieval-on-didemo","task":"Video Retrieval","dataset":"DiDeMo","model":"FROZEN","rank_in_archive_order":36,"of":40,"metrics":{"text-to-video Median Rank":"3","text-to-video R@1":"31.0","text-to-video R@10":"72.4","text-to-video R@5":"59.8"},"uses_additional_data":true},{"leaderboard":"/sota/video-retrieval-on-lsmdc","task":"Video Retrieval","dataset":"LSMDC","model":"FROZEN","rank_in_archive_order":27,"of":38,"metrics":{"text-to-video Median Rank":"20.0","text-to-video R@1":"15.0","text-to-video R@10":"39.8","text-to-video R@5":"30.8"},"uses_additional_data":true},{"leaderboard":"/sota/video-retrieval-on-msr-vtt","task":"Video Retrieval","dataset":"MSR-VTT","model":"FROZEN","rank_in_archive_order":23,"of":40,"metrics":{"text-to-video R@1":"32.5","text-to-video R@10":"71.2","text-to-video R@5":"61.5"},"uses_additional_data":false},{"leaderboard":"/sota/video-retrieval-on-msr-vtt-1ka","task":"Video Retrieval","dataset":"MSR-VTT-1kA","model":"FROZEN","rank_in_archive_order":49,"of":63,"metrics":{"text-to-video Median Rank":"3","text-to-video R@1":"31.0","text-to-video R@10":"70.5","text-to-video R@5":"59.5"},"uses_additional_data":true},{"leaderboard":"/sota/video-retrieval-on-msvd","task":"Video Retrieval","dataset":"MSVD","model":"FROZEN","rank_in_archive_order":22,"of":24,"metrics":{"text-to-video Median Rank":"3","text-to-video R@1":"33.7","text-to-video R@10":"76.3","text-to-video R@5":"64.7"},"uses_additional_data":true},{"leaderboard":"/sota/video-retrieval-on-queryd","task":"Video Retrieval","dataset":"QuerYD","model":"Frozen","rank_in_archive_order":4,"of":5,"metrics":{"text-to-video R@1":"53.8","text-to-video R@10":"82.7","text-to-video R@5":"75.7"},"uses_additional_data":true},{"leaderboard":"/sota/zero-shot-video-retrieval-on-didemo","task":"Zero-Shot Video Retrieval","dataset":"DiDeMo","model":"FROZEN","rank_in_archive_order":24,"of":26,"metrics":{"text-to-video R@1":"21.1","text-to-video R@10":"56.2","text-to-video R@5":"46.0"},"uses_additional_data":true},{"leaderboard":"/sota/zero-shot-video-retrieval-on-didemo","task":"Zero-Shot Video Retrieval","dataset":"DiDeMo","model":"M. Bain et. al.","rank_in_archive_order":25,"of":26,"metrics":{"text-to-video Median Rank":"7","text-to-video R@1":"20.2","text-to-video R@10":"58.5","text-to-video R@5":"46.4"},"uses_additional_data":false},{"leaderboard":"/sota/zero-shot-video-retrieval-on-msr-vtt","task":"Zero-Shot Video Retrieval","dataset":"MSR-VTT","model":"FROZEN","rank_in_archive_order":29,"of":41,"metrics":{"text-to-video Median Rank":"7.0","text-to-video R@1":"24.7","text-to-video R@10":"57.2","text-to-video R@5":"46.9"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2104.00650","atlas_url":"https://app.syntology.ai/?focus=2104.00650","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.00650"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/princetonvisualai/mqvr","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/m-bain/frozen-in-time","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/willard-yuan/video-text-retrieval-papers","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/m-bain/webvid","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/towhee-io/towhee","reach":null}],"summary":{"ran_fixture":1,"ran":2,"ran_draft_wrong":2,"unverified":6},"by_repo_kind":{"official":{"samples":11,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"544c604bc4f58d28","entry":"attn","repo":"m-bain/frozen-in-time","repo_kind":"official","path":"model/video_transformer.py","file_url":"https://github.com/m-bain/frozen-in-time/blob/HEAD/model/video_transformer.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"544c604bc4f58d28"}},{"code_sha256_prefix":"682226b68a0f1905","entry":"compute_similarity","repo":"m-bain/frozen-in-time","repo_kind":"official","path":"model/model.py","file_url":"https://github.com/m-bain/frozen-in-time/blob/HEAD/model/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"682226b68a0f1905"}},{"code_sha256_prefix":"9b0a787b92a87023","entry":"cosine_sim","repo":"m-bain/frozen-in-time","repo_kind":"official","path":"model/loss.py","file_url":"https://github.com/m-bain/frozen-in-time/blob/HEAD/model/loss.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9b0a787b92a87023"}},{"code_sha256_prefix":"185c7b880ca6135e","entry":"sim_matrix","repo":"m-bain/frozen-in-time","repo_kind":"official","path":"model/model.py","file_url":"https://github.com/m-bain/frozen-in-time/blob/HEAD/model/model.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"185c7b880ca6135e"}},{"code_sha256_prefix":"e50223362e6e9644","entry":"t2v_metrics","repo":"m-bain/frozen-in-time","repo_kind":"official","path":"model/metric.py","file_url":"https://github.com/m-bain/frozen-in-time/blob/HEAD/model/metric.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e50223362e6e9644"}},{"code_sha256_prefix":"8c2db3d4435816c0","entry":"nll_loss","repo":"m-bain/frozen-in-time","repo_kind":"official","path":"model/loss.py","file_url":"https://github.com/m-bain/frozen-in-time/blob/HEAD/model/loss.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8c2db3d4435816c0"}},{"code_sha256_prefix":"da16023b8239c604","entry":"order_sim","repo":"m-bain/frozen-in-time","repo_kind":"official","path":"model/loss.py","file_url":"https://github.com/m-bain/frozen-in-time/blob/HEAD/model/loss.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"da16023b8239c604"}},{"code_sha256_prefix":"ba8f16f08094547c","entry":"read_frames_cv2","repo":"m-bain/frozen-in-time","repo_kind":"official","path":"base/base_dataset.py","file_url":"https://github.com/m-bain/frozen-in-time/blob/HEAD/base/base_dataset.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ba8f16f08094547c"}},{"code_sha256_prefix":"63ecff4c1a42090b","entry":"retrieval_as_classification","repo":"m-bain/frozen-in-time","repo_kind":"official","path":"model/metric.py","file_url":"https://github.com/m-bain/frozen-in-time/blob/HEAD/model/metric.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"63ecff4c1a42090b"}},{"code_sha256_prefix":"4dadb89db8c00cc2","entry":"sample_frames","repo":"m-bain/frozen-in-time","repo_kind":"official","path":"base/base_dataset.py","file_url":"https://github.com/m-bain/frozen-in-time/blob/HEAD/base/base_dataset.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4dadb89db8c00cc2"}},{"code_sha256_prefix":"000e15329b7114d0","entry":"v2t_metrics","repo":"m-bain/frozen-in-time","repo_kind":"official","path":"model/metric.py","file_url":"https://github.com/m-bain/frozen-in-time/blob/HEAD/model/metric.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"000e15329b7114d0"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}