{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/connecting-vision-and-language-with-video","title":"Connecting Vision and Language with Video Localized Narratives","arxiv_id":"2302.11217","date":"2023-02-22","proceeding":"CVPR 2023 1","authors":["Paul Voigtlaender","Soravit Changpinyo","Jordi Pont-Tuset","Radu Soricut","Vittorio Ferrari"],"abstract":"We propose Video Localized Narratives, a new form of multimodal video annotations connecting vision and language. In the original Localized Narratives, annotators speak and move their mouse simultaneously on an image, thus grounding each word with a mouse trace segment. However, this is challenging on a video. Our new protocol empowers annotators to tell the story of a video with Localized Narratives, capturing even complex events involving multiple actors interacting with each other and with several passive objects. We annotated 20k videos of the OVIS, UVO, and Oops datasets, totalling 1.7M words. Based on this data, we also construct new benchmarks for the video narrative grounding and video question answering tasks, and provide reference results from strong baseline models. Our annotations are available at https://google.github.io/video-localized-narratives/.","url_abs":"https://arxiv.org/abs/2302.11217v2","url_pdf":"https://arxiv.org/pdf/2302.11217v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"connecting-vision-and-language-with-video","repo_url":"https://github.com/google/video-localized-narratives","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"video-narrative-grounding","task_name":"Video Narrative Grounding"},{"task_slug":"video-question-answering","task_name":"Video Question Answering"}],"methods":[],"datasets_introduced":[{"slug":"video-localized-narratives","name":"Video Localized Narratives","full_name":""}],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2302.11217","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2302.11217"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/google/video-localized-narratives","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"unverified":4},"by_repo_kind":{"official":{"samples":4,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"ac3749fc8a1bc60f","entry":"accuracy","repo":"google/video-localized-narratives","repo_kind":"official","path":"video_localized_narratives/videoqa/text_output/eval_text_output.py","file_url":"https://github.com/google/video-localized-narratives/blob/HEAD/video_localized_narratives/videoqa/text_output/eval_text_output.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ac3749fc8a1bc60f"}},{"code_sha256_prefix":"824ac285c53c6908","entry":"box_mask_from_bounding_box","repo":"google/video-localized-narratives","repo_kind":"official","path":"video_localized_narratives/videoqa/location_output/eval_utils.py","file_url":"https://github.com/google/video-localized-narratives/blob/HEAD/video_localized_narratives/videoqa/location_output/eval_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"824ac285c53c6908"}},{"code_sha256_prefix":"8db471ea0b45d04c","entry":"estimate_approximate_square_gt_mask","repo":"google/video-localized-narratives","repo_kind":"official","path":"video_localized_narratives/videoqa/location_output/square_prediction.py","file_url":"https://github.com/google/video-localized-narratives/blob/HEAD/video_localized_narratives/videoqa/location_output/square_prediction.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8db471ea0b45d04c"}},{"code_sha256_prefix":"c8943844160cca50","entry":"evaluate_result","repo":"google/video-localized-narratives","repo_kind":"official","path":"video_localized_narratives/videoqa/location_output/eval_utils.py","file_url":"https://github.com/google/video-localized-narratives/blob/HEAD/video_localized_narratives/videoqa/location_output/eval_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c8943844160cca50"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}