{"url":"/dataset/multisubs","name":"MultiSubs","full_name":"MultiSubs: A Large-scale Multimodal and Multilingual Dataset","description_markdown":"MultiSubs is a dataset of multilingual subtitles gathered from [the OPUS OpenSubtitles dataset](https://opus.nlpl.eu/OpenSubtitles.php), which in turn was sourced from [opensubtitles.org](http://www.opensubtitles.org/). We have supplemented some text fragments (visually salient nouns in this release) within the subtitles with web images, where the word sense of the fragment has been disambiguated using a cross-lingual approach. We have introduced a fill-in-the-blank task and a lexical translation task to demonstrate the utility of the dataset. Please refer to [our paper](https://arxiv.org/abs/2103.01910) for a more detailed description of the dataset and tasks. Multisubs will benefit research on visual grounding of words especially in the context of free-form sentence.\n\nJosiah Wang, Pranava Madhyastha, Josiel Figueiredo, Chiraag Lala, Lucia Specia (2021). [MultiSubs: A Large-scale Multimodal and Multilingual Dataset](https://arxiv.org/abs/2103.01910). CoRR, abs/2103.01910. Available at: [https://arxiv.org/abs/2103.01910]\n(https://arxiv.org/abs/2103.01910)","description_withheld":null,"homepage":"https://doi.org/10.5281/zenodo.5034604","introduced_date":"2021-06-30","introduced_date_note":null,"introduced_by":{"paper":"/paper/multisubs-a-large-scale-multimodal-and","title":"MultiSubs: A Large-scale Multimodal and Multilingual Dataset","first_author":"Josiah Wang","url":null},"license":{"name":"Creative Commons Attribution 4.0 International","url":"https://creativecommons.org/licenses/by/4.0/legalcode"},"modalities":[{"name":"Images","url":"/datasets/modality/images"},{"name":"Texts","url":"/datasets/modality/texts"}],"tasks":[{"name":"Slot Filling","url":"/task/slot-filling","datasets_with_task":"/datasets/task/slot-filling"},{"name":"Multimodal Lexical Translation","url":"/task/multimodal-lexical-translation","datasets_with_task":"/datasets/task/multimodal-lexical-translation"},{"name":"Multimodal Text Prediction","url":"/task/multimodal-text-prediction","datasets_with_task":"/datasets/task/multimodal-text-prediction"}],"languages":[{"name":"English","url":"/datasets/language/english"},{"name":"French","url":"/datasets/language/french"},{"name":"Spanish","url":"/datasets/language/spanish"},{"name":"German","url":"/datasets/language/german"},{"name":"Portuguese","url":"/datasets/language/portuguese"}],"variants":["MultiSubs","MultiSubs English-Spanish","MultiSubs English-Portuguese","MultiSubs English-French","MultiSubs English-German"],"data_loaders":[],"num_papers_in_archive":4,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28"},"benchmarks":[{"leaderboard":"/sota/multimodal-lexical-translation-on-multisubs","task":"Multimodal Lexical Translation","dataset_variant":"MultiSubs English-Spanish","rows":1,"metrics":["ALI"],"first_row_in_archive_order":{"model":"Multimodal BRNN","paper":"/paper/multisubs-a-large-scale-multimodal-and","metrics":{"ALI":"0.81"},"code_links":[{"title":"josiahwang/multisubs-eval","url":"https://github.com/josiahwang/multisubs-eval"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multimodal-lexical-translation-on-multisubs-1","task":"Multimodal Lexical Translation","dataset_variant":"MultiSubs English-Portuguese","rows":1,"metrics":["ALI"],"first_row_in_archive_order":{"model":"Multimodal BRNN","paper":"/paper/multisubs-a-large-scale-multimodal-and","metrics":{"ALI":"0.80"},"code_links":[{"title":"josiahwang/multisubs-eval","url":"https://github.com/josiahwang/multisubs-eval"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multimodal-lexical-translation-on-multisubs-2","task":"Multimodal Lexical Translation","dataset_variant":"MultiSubs English-French","rows":1,"metrics":["ALI"],"first_row_in_archive_order":{"model":"Multimodal BRNN","paper":"/paper/multisubs-a-large-scale-multimodal-and","metrics":{"ALI":"0.81"},"code_links":[{"title":"josiahwang/multisubs-eval","url":"https://github.com/josiahwang/multisubs-eval"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multimodal-lexical-translation-on-multisubs-3","task":"Multimodal Lexical Translation","dataset_variant":"MultiSubs English-German","rows":1,"metrics":["ALI"],"first_row_in_archive_order":{"model":"Multimodal BRNN","paper":"/paper/multisubs-a-large-scale-multimodal-and","metrics":{"ALI":"0.94"},"code_links":[{"title":"josiahwang/multisubs-eval","url":"https://github.com/josiahwang/multisubs-eval"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/multimodal-text-prediction-on-multisubs","task":"Multimodal Text Prediction","dataset_variant":"MultiSubs","rows":1,"metrics":["Accuracy","Word similarity"],"first_row_in_archive_order":{"model":"9-gram LM with back-off","paper":"/paper/multisubs-a-large-scale-multimodal-and","metrics":{"Accuracy":"30.35","Word similarity":"0.44"},"code_links":[{"title":"josiahwang/multisubs-eval","url":"https://github.com/josiahwang/multisubs-eval"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"}],"papers_with_a_benchmark_row":[{"paper":"/paper/multisubs-a-large-scale-multimodal-and","title":"MultiSubs: A Large-scale Multimodal and Multilingual Dataset","date":"2021-03-02","rows_on_this_dataset":5,"code_links":1,"syntology":null}],"syntology_totals":{"read_at":"2026-09-24T18:15:14+00:00","papers_with_samples":0,"samples_harvested":0,"samples_ran":0,"samples_unverified":0,"pointer_only_for_licence":0,"papers_with_no_sample_that_ran":0,"note":"the per-paper counts above, summed; not a rate"},"papers_note":"The archive never published its papers-using-dataset list; these are papers with a leaderboard row on this dataset's benchmarks."}