{"url":"/dataset/localized-narratives","name":"Localized Narratives","full_name":null,"description_markdown":"We propose Localized Narratives, a new form of multimodal image annotations connecting vision and language. We ask annotators to describe an image with their voice while simultaneously hovering their mouse over the region they are describing. Since the voice and the mouse pointer are synchronized, we can localize every single word in the description. This dense visual grounding takes the form of a mouse trace segment per word and is unique to our data. We annotated 849k images with Localized Narratives: the whole COCO, Flickr30k, and ADE20K datasets, and 671k images of Open Images, all of which we make publicly available. We provide an extensive analysis of these annotations showing they are diverse, accurate, and efficient to produce. We also demonstrate their utility on the application of controlled image captioning.","description_withheld":null,"homepage":"https://google.github.io/localized-narratives/","introduced_date":"2019-12-06","introduced_date_note":null,"introduced_by":{"paper":"/paper/connecting-vision-and-language-with-localized","title":"Connecting Vision and Language with Localized Narratives","first_author":"Jordi Pont-Tuset","url":null},"license":null,"modalities":[{"name":"Images","url":"/datasets/modality/images"},{"name":"Texts","url":"/datasets/modality/texts"},{"name":"Audio","url":"/datasets/modality/audio"}],"tasks":[{"name":"Image Retrieval","url":"/task/image-retrieval","datasets_with_task":"/datasets/task/image-retrieval"},{"name":"Image Captioning","url":"/task/image-captioning","datasets_with_task":"/datasets/task/image-captioning"},{"name":"Text to Audio Retrieval","url":"/task/text-to-audio-retrieval","datasets_with_task":"/datasets/task/text-to-audio-retrieval"},{"name":"Audio to Text Retrieval","url":"/task/audio-to-text-retrieval","datasets_with_task":"/datasets/task/audio-to-text-retrieval"}],"languages":[{"name":"English","url":"/datasets/language/english"}],"variants":["Localized Narratives"],"data_loaders":[],"num_papers_in_archive":63,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28"},"benchmarks":[{"leaderboard":"/sota/image-captioning-on-localized-narratives","task":"Image Captioning","dataset_variant":"Localized Narratives","rows":2,"metrics":["CIDEr"],"first_row_in_archive_order":{"model":"LoopCAG","paper":"/paper/control-image-captioning-spatially-and","metrics":{"CIDEr":"114.0"},"code_links":[]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/image-retrieval-on-localized-narratives","task":"Image Retrieval","dataset_variant":"Localized Narratives","rows":1,"metrics":["Text-to-image R@1","Text-to-image R@10","Text-to-image R@5"],"first_row_in_archive_order":{"model":"OPT","paper":"/paper/opt-omni-perception-pre-trainer-for-cross","metrics":{"Text-to-image R@1":"0.4196","Text-to-image R@10":"0.8126","Text-to-image R@5":"0.72"},"code_links":[{"title":"mindspore-ai/models","url":"https://github.com/mindspore-ai/models/tree/master/research/mm/opt"},{"title":"2023-MindSpore-1/ms-code-161","url":"https://github.com/2023-MindSpore-1/ms-code-161"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/text-to-audio-retrieval-on-localized","task":"Text to Audio Retrieval","dataset_variant":"Localized Narratives","rows":1,"metrics":["Text-to-audio R@1","Text-to-audio R@10","Text-to-audio R@5"],"first_row_in_archive_order":{"model":"OPT","paper":"/paper/opt-omni-perception-pre-trainer-for-cross","metrics":{"Text-to-audio R@1":"0.78","Text-to-audio R@10":"0.958","Text-to-audio R@5":"0.927"},"code_links":[{"title":"mindspore-ai/models","url":"https://github.com/mindspore-ai/models/tree/master/research/mm/opt"},{"title":"2023-MindSpore-1/ms-code-161","url":"https://github.com/2023-MindSpore-1/ms-code-161"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"}],"papers_with_a_benchmark_row":[{"paper":"/paper/control-image-captioning-spatially-and","title":"Control Image Captioning Spatially and Temporally","date":"2021-08-01","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/opt-omni-perception-pre-trainer-for-cross","title":"OPT: Omni-Perception Pre-Trainer for Cross-Modal Understanding and Generation","date":"2021-07-01","rows_on_this_dataset":2,"code_links":2,"syntology":null},{"paper":"/paper/connecting-vision-and-language-with-localized","title":"Connecting Vision and Language with Localized Narratives","date":"2019-12-06","rows_on_this_dataset":1,"code_links":1,"syntology":null}],"syntology_totals":{"read_at":"2026-09-24T18:15:14+00:00","papers_with_samples":0,"samples_harvested":0,"samples_ran":0,"samples_unverified":0,"pointer_only_for_licence":0,"papers_with_no_sample_that_ran":0,"note":"the per-paper counts above, summed; not a rate"},"papers_note":"The archive never published its papers-using-dataset list; these are papers with a leaderboard row on this dataset's benchmarks."}