{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/revealing-single-frame-bias-for-video-and","title":"Revealing Single Frame Bias for Video-and-Language Learning","arxiv_id":"2206.03428","date":"2022-06-07","proceeding":null,"authors":["Jie Lei","Tamara L. Berg","Mohit Bansal"],"abstract":"Training an effective video-and-language model intuitively requires multiple frames as model inputs. However, it is unclear whether using multiple frames is beneficial to downstream tasks, and if yes, whether the performance gain is worth the drastically-increased computation and memory costs resulting from using more frames. In this work, we explore single-frame models for video-and-language learning. On a diverse set of video-and-language tasks (including text-to-video retrieval and video question answering), we show the surprising result that, with large-scale pre-training and a proper frame ensemble strategy at inference time, a single-frame trained model that does not consider temporal information can achieve better performance than existing methods that use multiple frames for training. This result reveals the existence of a strong \"static appearance bias\" in popular video-and-language datasets. Therefore, to allow for a more comprehensive evaluation of video-and-language models, we propose two new retrieval tasks based on existing fine-grained action recognition datasets that encourage temporal modeling. Our code is available at https://github.com/jayleicn/singularity","url_abs":"https://arxiv.org/abs/2206.03428v1","url_pdf":"https://arxiv.org/pdf/2206.03428v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"revealing-single-frame-bias-for-video-and","repo_url":"https://github.com/jayleicn/singularity","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"revealing-single-frame-bias-for-video-and","repo_url":"https://github.com/jayleicn/ClipBERT","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"action-recognition-in-videos","task_name":"Action Recognition"},{"task_slug":"fine-grained-action-recognition","task_name":"Fine-grained Action Recognition"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"text-to-video-retrieval","task_name":"Text to Video Retrieval"},{"task_slug":"video-question-answering","task_name":"Video Question Answering"},{"task_slug":"video-retrieval","task_name":"Video Retrieval"},{"task_slug":"zero-shot-video-retrieval","task_name":"Zero-Shot Video Retrieval"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/video-question-answering-on-activitynet-qa","task":"Video Question Answering","dataset":"ActivityNet-QA","model":"Singularity-temporal","rank_in_archive_order":20,"of":36,"metrics":{"Accuracy":"44.1"},"uses_additional_data":true},{"leaderboard":"/sota/video-question-answering-on-activitynet-qa","task":"Video Question Answering","dataset":"ActivityNet-QA","model":"Singularity","rank_in_archive_order":22,"of":36,"metrics":{"Accuracy":"43.1"},"uses_additional_data":true},{"leaderboard":"/sota/video-question-answering-on-msrvtt-mc","task":"Video Question Answering","dataset":"MSRVTT-MC","model":"Singularity-temporal","rank_in_archive_order":5,"of":7,"metrics":{"Accuracy":"93.7"},"uses_additional_data":false},{"leaderboard":"/sota/video-question-answering-on-msrvtt-mc","task":"Video Question Answering","dataset":"MSRVTT-MC","model":"Singularity","rank_in_archive_order":7,"of":7,"metrics":{"Accuracy":"92.1"},"uses_additional_data":false},{"leaderboard":"/sota/video-question-answering-on-msrvtt-qa","task":"Video Question Answering","dataset":"MSRVTT-QA","model":"Singularity-temporal","rank_in_archive_order":12,"of":14,"metrics":{"Accuracy":"43.9"},"uses_additional_data":false},{"leaderboard":"/sota/video-question-answering-on-msrvtt-qa","task":"Video Question Answering","dataset":"MSRVTT-QA","model":"Singularity","rank_in_archive_order":13,"of":14,"metrics":{"Accuracy":"43.5"},"uses_additional_data":false},{"leaderboard":"/sota/video-retrieval-on-activitynet","task":"Video Retrieval","dataset":"ActivityNet","model":"Singularity","rank_in_archive_order":19,"of":31,"metrics":{"text-to-video R@1":"47.1","text-to-video R@10":"85.5","text-to-video R@5":"75.5"},"uses_additional_data":true},{"leaderboard":"/sota/video-retrieval-on-didemo","task":"Video Retrieval","dataset":"DiDeMo","model":"Singularity","rank_in_archive_order":18,"of":40,"metrics":{"text-to-video R@1":"53.9","text-to-video R@10":"86.9","text-to-video R@5":"79.4"},"uses_additional_data":true},{"leaderboard":"/sota/video-retrieval-on-msr-vtt-1ka","task":"Video Retrieval","dataset":"MSR-VTT-1kA","model":"Singularity","rank_in_archive_order":37,"of":63,"metrics":{"text-to-video R@1":"41.5","text-to-video R@10":"77","text-to-video R@5":"68.7"},"uses_additional_data":true},{"leaderboard":"/sota/video-retrieval-on-ssv2-label-retrieval","task":"Video Retrieval","dataset":"SSv2-label retrieval","model":"Singularity-temporal","rank_in_archive_order":5,"of":5,"metrics":{"text-to-video R@1":"47.4","text-to-video R@10":"84","text-to-video R@5":"75.9"},"uses_additional_data":true},{"leaderboard":"/sota/video-retrieval-on-ssv2-template-retrieval","task":"Video Retrieval","dataset":"SSv2-template retrieval","model":"Singularity-temporal","rank_in_archive_order":5,"of":5,"metrics":{"text-to-video R@1":"77.6","text-to-video R@10":"98.9","text-to-video R@5":"96"},"uses_additional_data":true},{"leaderboard":"/sota/zero-shot-video-retrieval-on-activitynet","task":"Zero-Shot Video Retrieval","dataset":"ActivityNet","model":"Singularity-temporal-5M","rank_in_archive_order":10,"of":12,"metrics":{"text-to-video R@1":"30.8","text-to-video R@10":"66.3","text-to-video R@5":"55.9"},"uses_additional_data":true},{"leaderboard":"/sota/zero-shot-video-retrieval-on-activitynet","task":"Zero-Shot Video Retrieval","dataset":"ActivityNet","model":"Singularity-temporal-17M","rank_in_archive_order":12,"of":12,"metrics":{"text-to-video R@1":"30.6","text-to-video R@10":"66.9","text-to-video R@5":"55.6"},"uses_additional_data":true},{"leaderboard":"/sota/zero-shot-video-retrieval-on-didemo","task":"Zero-Shot Video Retrieval","dataset":"DiDeMo","model":"Singularity-17M","rank_in_archive_order":11,"of":26,"metrics":{"text-to-video R@1":"37.1","text-to-video R@10":"69.9","text-to-video R@5":"61.7"},"uses_additional_data":true},{"leaderboard":"/sota/zero-shot-video-retrieval-on-didemo","task":"Zero-Shot Video Retrieval","dataset":"DiDeMo","model":"Singularity-5M","rank_in_archive_order":12,"of":26,"metrics":{"text-to-video R@1":"36.9","text-to-video R@10":"69.3","text-to-video R@5":"61.1"},"uses_additional_data":true},{"leaderboard":"/sota/zero-shot-video-retrieval-on-msr-vtt","task":"Zero-Shot Video Retrieval","dataset":"MSR-VTT","model":"Singularity-17M","rank_in_archive_order":20,"of":41,"metrics":{"text-to-video R@1":"34.0","text-to-video R@10":"66.7","text-to-video R@5":"56.7"},"uses_additional_data":true},{"leaderboard":"/sota/zero-shot-video-retrieval-on-msr-vtt","task":"Zero-Shot Video Retrieval","dataset":"MSR-VTT","model":"Singularity-5M","rank_in_archive_order":24,"of":41,"metrics":{"text-to-video R@1":"28.4","text-to-video R@10":"59.5","text-to-video R@5":"50.2"},"uses_additional_data":true}],"syntology":{"syntology_url":"https://syntology.ai/paper/2206.03428","atlas_url":"https://app.syntology.ai/?focus=2206.03428","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2206.03428"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jayleicn/ClipBERT","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jayleicn/singularity","reach":null}],"summary":{"ran":8,"ran_draft_wrong":1,"unverified":3},"by_repo_kind":{"listed":{"samples":12,"ran":9,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"17e89eef72e37efe","entry":"conv3x3","repo":"jayleicn/ClipBERT","repo_kind":"listed","path":"src/modeling/grid_feat.py","file_url":"https://github.com/jayleicn/ClipBERT/blob/HEAD/src/modeling/grid_feat.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"17e89eef72e37efe"}},{"code_sha256_prefix":"0cf387da46369b1a","entry":"get_padding","repo":"jayleicn/ClipBERT","repo_kind":"listed","path":"src/datasets/data_utils.py","file_url":"https://github.com/jayleicn/ClipBERT/blob/HEAD/src/datasets/data_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0cf387da46369b1a"}},{"code_sha256_prefix":"f31be4ddaa3ec6c2","entry":"get_random_sample_indices","repo":"jayleicn/ClipBERT","repo_kind":"listed","path":"src/modeling/modeling.py","file_url":"https://github.com/jayleicn/ClipBERT/blob/HEAD/src/modeling/modeling.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f31be4ddaa3ec6c2"}},{"code_sha256_prefix":"707f6cd75833c65d","entry":"get_start_end_idx","repo":"jayleicn/ClipBERT","repo_kind":"listed","path":"src/datasets/decoder.py","file_url":"https://github.com/jayleicn/ClipBERT/blob/HEAD/src/datasets/decoder.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"707f6cd75833c65d"}},{"code_sha256_prefix":"b6331823c359e9d8","entry":"get_video_decoding_kwargs","repo":"jayleicn/ClipBERT","repo_kind":"listed","path":"src/datasets/dataset_base.py","file_url":"https://github.com/jayleicn/ClipBERT/blob/HEAD/src/datasets/dataset_base.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b6331823c359e9d8"}},{"code_sha256_prefix":"e9242c50098a6ea2","entry":"image_to_tensor","repo":"jayleicn/ClipBERT","repo_kind":"listed","path":"src/datasets/data_utils.py","file_url":"https://github.com/jayleicn/ClipBERT/blob/HEAD/src/datasets/data_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e9242c50098a6ea2"}},{"code_sha256_prefix":"f828f173e6c2fea2","entry":"mask_batch_text_tokens","repo":"jayleicn/ClipBERT","repo_kind":"listed","path":"src/datasets/data_utils.py","file_url":"https://github.com/jayleicn/ClipBERT/blob/HEAD/src/datasets/data_utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f828f173e6c2fea2"}},{"code_sha256_prefix":"b68769dd52ad6c44","entry":"move_to_cuda","repo":"jayleicn/ClipBERT","repo_kind":"listed","path":"src/datasets/dataloader.py","file_url":"https://github.com/jayleicn/ClipBERT/blob/HEAD/src/datasets/dataloader.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b68769dd52ad6c44"}},{"code_sha256_prefix":"f307b4793e8204b7","entry":"temporal_sampling","repo":"jayleicn/ClipBERT","repo_kind":"listed","path":"src/datasets/decoder.py","file_url":"https://github.com/jayleicn/ClipBERT/blob/HEAD/src/datasets/decoder.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f307b4793e8204b7"}},{"code_sha256_prefix":"5a4d93d8be97ccd9","entry":"instance_bce_with_logits","repo":"jayleicn/ClipBERT","repo_kind":"listed","path":"src/modeling/modeling.py","file_url":"https://github.com/jayleicn/ClipBERT/blob/HEAD/src/modeling/modeling.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5a4d93d8be97ccd9"}},{"code_sha256_prefix":"3e81145412a40e36","entry":"parse_with_config","repo":"jayleicn/ClipBERT","repo_kind":"listed","path":"src/configs/config.py","file_url":"https://github.com/jayleicn/ClipBERT/blob/HEAD/src/configs/config.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3e81145412a40e36"}},{"code_sha256_prefix":"b28b46ef7776c434","entry":"pyav_decode_stream","repo":"jayleicn/ClipBERT","repo_kind":"listed","path":"src/datasets/decoder.py","file_url":"https://github.com/jayleicn/ClipBERT/blob/HEAD/src/datasets/decoder.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b28b46ef7776c434"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}