{"url":"/task/multimodal-machine-translation","name":"Multimodal Machine Translation","slug":"multimodal-machine-translation","description_markdown":"Multimodal machine translation is the task of doing machine translation with multiple data sources - for example, translating \"a bird is flying over water\" + an image of a bird over water to German text.\r\n\r\n<span style=\"color:grey; opacity: 0.6\">( Image credit: [Findings of the Third Shared Task on Multimodal Machine Translation](https://www.aclweb.org/anthology/W18-6402.pdf) )</span>","categories":[{"name":"Computer Vision","url":"/area/computer-vision"},{"name":"Natural Language Processing","url":"/area/natural-language-processing"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":108,"papers_with_code":38,"benchmarks":3,"benchmark_tables_in_archive":3,"benchmark_tables_shown":3,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":5,"subtasks":2,"parent_tasks":1},"benchmarks":[{"leaderboard":"/sota/multimodal-machine-translation-on-multi30k","slug":"multimodal-machine-translation-on-multi30k","dataset":"Multi30K","dataset_url":"/dataset/multi30k","rows_in_archive":15,"metrics":["BLEU (EN-DE)","BLUE (DE-EN)","Meteor (EN-DE)","Meteor (EN-FR)"],"first_row_in_archive_order":{"model":"ERNIE-UniX2","paper_title":"ERNIE-UniX2: A Unified Cross-lingual Cross-modal Framework for Understanding and Generation","paper_url":"/paper/ernie-unix2-a-unified-cross-lingual-cross","paper_date":"2022-11-09","arxiv_id":"2211.04861","code_links":[],"syntology":null}},{"leaderboard":"/sota/multimodal-machine-translation-on-hindi-1","slug":"multimodal-machine-translation-on-hindi-1","dataset":"Hindi Visual Genome (Test Set)","dataset_url":null,"rows_in_archive":1,"metrics":["BLEU (EN-HI)"],"first_row_in_archive_order":{"model":"ViTA","paper_title":"ViTA: Visual-Linguistic Translation by Aligning Object Tags","paper_url":"/paper/vita-visual-linguistic-translation-by","paper_date":"2021-06-01","arxiv_id":"2106.00250","code_links":[{"title":"kshitij98/vita","url":"https://github.com/kshitij98/vita"}],"syntology":null}},{"leaderboard":"/sota/multimodal-machine-translation-on-hindi-2","slug":"multimodal-machine-translation-on-hindi-2","dataset":"Hindi Visual Genome (Challenge Set)","dataset_url":null,"rows_in_archive":1,"metrics":["BLEU (EN-HI)"],"first_row_in_archive_order":{"model":"ViTA","paper_title":"ViTA: Visual-Linguistic Translation by Aligning Object Tags","paper_url":"/paper/vita-visual-linguistic-translation-by","paper_date":"2021-06-01","arxiv_id":"2106.00250","code_links":[{"title":"kshitij98/vita","url":"https://github.com/kshitij98/vita"}],"syntology":null}}],"datasets":[{"url":"/dataset/multi30k","name":"Multi30K","full_name":"","num_papers_in_archive":139},{"url":"/dataset/hindi-visual-genome","name":"Hindi Visual Genome","full_name":"","num_papers_in_archive":7},{"url":"/dataset/havg","name":"HaVG","full_name":"Hausa Visual Genome Dataset","num_papers_in_archive":3},{"url":"/dataset/wmt-2016-biomedical","name":"WMT 2016 Biomedical","full_name":"WMT 2016 Biomedical Translation Task","num_papers_in_archive":1},{"url":"/dataset/wmt-2016-it","name":"WMT 2016 IT","full_name":"WMT 2016 IT Translation Task","num_papers_in_archive":1}],"subtasks":[{"url":"/task/face-to-face-translation","name":"Face to Face Translation"},{"url":"/task/multimodal-lexical-translation","name":"Multimodal Lexical Translation"}],"parent_tasks":[{"url":"/task/machine-translation","name":"Machine Translation"}],"papers":{"order":"repositories listed in the archive (desc), then date (desc); the archive holds no stars","population":"papers tagged with this task that list at least one repository in the archive","shown":30,"of":38,"tagged_in_all":108,"items":[{"url":"/paper/attention-is-all-you-need","title":"Attention Is All You Need","date":"2017-06-12","arxiv_id":"1706.03762","repositories_listed":595,"syntology":{"n":946,"n_ran":610,"n_unverified":336,"n_pointer_only":451}},{"url":"/paper/towards-zero-shot-multimodal-machine","title":"Towards Zero-Shot Multimodal Machine Translation","date":"2024-07-18","arxiv_id":"2407.13579","repositories_listed":2,"syntology":null},{"url":"/paper/tackling-ambiguity-with-images-improved","title":"Tackling Ambiguity with Images: Improved Multimodal Machine Translation and Contrastive Evaluation","date":"2022-12-20","arxiv_id":"2212.10140","repositories_listed":2,"syntology":{"n":2,"n_ran":2,"n_unverified":0,"n_pointer_only":2}},{"url":"/paper/on-vision-features-in-multimodal-machine-1","title":"On Vision Features in Multimodal Machine Translation","date":"2022-03-17","arxiv_id":"2203.09173","repositories_listed":2,"syntology":{"n":3,"n_ran":2,"n_unverified":1,"n_pointer_only":0}},{"url":"/paper/multi30k-multilingual-english-german-image","title":"Multi30K: Multilingual English-German Image Descriptions","date":"2016-05-02","arxiv_id":"1605.00459","repositories_listed":2,"syntology":null},{"url":"/paper/3am-an-ambiguity-aware-multi-modal-machine","title":"3AM: An Ambiguity-Aware Multi-Modal Machine Translation Dataset","date":"2024-04-29","arxiv_id":"2404.18413","repositories_listed":1,"syntology":null},{"url":"/paper/seamless-multilingual-expressive-and","title":"Seamless: Multilingual Expressive and Streaming Speech Translation","date":"2023-12-08","arxiv_id":"2312.05187","repositories_listed":1,"syntology":null},{"url":"/paper/video-helpful-multimodal-machine-translation","title":"Video-Helpful Multimodal Machine Translation","date":"2023-10-31","arxiv_id":"2310.20201","repositories_listed":1,"syntology":null},{"url":"/paper/incorporating-probing-signals-into-multimodal","title":"Incorporating Probing Signals into Multimodal Machine Translation via Visual Question-Answering Pairs","date":"2023-10-26","arxiv_id":"2310.17133","repositories_listed":1,"syntology":null},{"url":"/paper/bridging-the-gap-between-synthetic-and","title":"Bridging the Gap between Synthetic and Authentic Images for Multimodal Machine Translation","date":"2023-10-20","arxiv_id":"2310.13361","repositories_listed":1,"syntology":null},{"url":"/paper/cliptrans-transferring-visual-knowledge-with","title":"CLIPTrans: Transferring Visual Knowledge with Pre-trained Models for Multimodal Machine Translation","date":"2023-08-29","arxiv_id":"2308.15226","repositories_listed":1,"syntology":{"n":7,"n_ran":5,"n_unverified":2,"n_pointer_only":7}},{"url":"/paper/havqa-a-dataset-for-visual-question-answering","title":"HaVQA: A Dataset for Visual Question Answering and Multimodal Research in Hausa Language","date":"2023-05-28","arxiv_id":"2305.17690","repositories_listed":1,"syntology":null},{"url":"/paper/bigvideo-a-large-scale-video-subtitle","title":"BigVideo: A Large-scale Video Subtitle Translation Dataset for Multimodal Machine Translation","date":"2023-05-23","arxiv_id":"2305.18326","repositories_listed":1,"syntology":null},{"url":"/paper/scene-graph-as-pivoting-inference-time-image","title":"Scene Graph as Pivoting: Inference-time Image-free Unsupervised Multimodal Machine Translation with Visual Scene Hallucination","date":"2023-05-20","arxiv_id":"2305.12256","repositories_listed":1,"syntology":null},{"url":"/paper/beyond-triplet-leveraging-the-most-data-for","title":"Beyond Triplet: Leveraging the Most Data for Multimodal Machine Translation","date":"2022-12-20","arxiv_id":"2212.10313","repositories_listed":1,"syntology":null},{"url":"/paper/distill-the-image-to-nowhere-inversion","title":"Distill the Image to Nowhere: Inversion Knowledge Distillation for Multimodal Machine Translation","date":"2022-10-10","arxiv_id":"2210.04468","repositories_listed":1,"syntology":{"n":5,"n_ran":4,"n_unverified":1,"n_pointer_only":5}},{"url":"/paper/valhalla-visual-hallucination-for-machine","title":"VALHALLA: Visual Hallucination for Machine Translation","date":"2022-05-31","arxiv_id":"2206.00100","repositories_listed":1,"syntology":{"n":10,"n_ran":10,"n_unverified":0,"n_pointer_only":0}},{"url":"/paper/neural-machine-translation-with-phrase-level","title":"Neural Machine Translation with Phrase-Level Universal Visual Representations","date":"2022-03-19","arxiv_id":"2203.10299","repositories_listed":1,"syntology":{"n":1,"n_ran":0,"n_unverified":1,"n_pointer_only":1}},{"url":"/paper/msctd-a-multimodal-sentiment-chat-translation","title":"MSCTD: A Multimodal Sentiment Chat Translation Dataset","date":"2022-02-28","arxiv_id":"2202.13645","repositories_listed":1,"syntology":null},{"url":"/paper/visa-an-ambiguous-subtitles-dataset-for","title":"VISA: An Ambiguous Subtitles Dataset for Visual Scene-Aware Machine Translation","date":"2022-01-20","arxiv_id":"2201.08054","repositories_listed":1,"syntology":null},{"url":"/paper/vision-matters-when-it-should-sanity-checking","title":"Vision Matters When It Should: Sanity Checking Multimodal Machine Translation Models","date":"2021-09-08","arxiv_id":"2109.03415","repositories_listed":1,"syntology":null},{"url":"/paper/bertgen-multi-task-generation-through-bert","title":"BERTGEN: Multi-task Generation through BERT","date":"2021-06-07","arxiv_id":"2106.03484","repositories_listed":1,"syntology":null},{"url":"/paper/vita-visual-linguistic-translation-by","title":"ViTA: Visual-Linguistic Translation by Aligning Object Tags","date":"2021-06-01","arxiv_id":"2106.00250","repositories_listed":1,"syntology":null},{"url":"/paper/cultural-and-geographical-influences-on-image","title":"Cultural and Geographical Influences on Image Translatability of Words across Languages","date":"2021-06-01","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/cross-lingual-visual-pre-training-for","title":"Cross-lingual Visual Pre-training for Multimodal Machine Translation","date":"2021-01-25","arxiv_id":"2101.10044","repositories_listed":1,"syntology":null},{"url":"/paper/dynamic-context-guided-capsule-network-for","title":"Dynamic Context-guided Capsule Network for Multimodal Machine Translation","date":"2020-09-04","arxiv_id":"2009.02016","repositories_listed":1,"syntology":null},{"url":"/paper/multimodal-transformer-for-multimodal-machine","title":"Multimodal Transformer for Multimodal Machine Translation","date":"2020-07-01","arxiv_id":null,"repositories_listed":1,"syntology":null},{"url":"/paper/self-knowledge-distillation-a-simple-way-for","title":"Self-Knowledge Distillation with Progressive Refinement of Targets","date":"2020-06-22","arxiv_id":"2006.12000","repositories_listed":1,"syntology":{"n":5,"n_ran":4,"n_unverified":1,"n_pointer_only":0}},{"url":"/paper/m3p-learning-universal-representations-via","title":"M3P: Learning Universal Representations via Multitask Multilingual Multimodal Pre-training","date":"2020-06-04","arxiv_id":"2006.02635","repositories_listed":1,"syntology":null},{"url":"/paper/distilling-translations-with-visual-awareness","title":"Distilling Translations with Visual Awareness","date":"2019-06-18","arxiv_id":"1906.07701","repositories_listed":1,"syntology":null}],"syntology_records":8,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-09-25T09:33:49+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"}},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"}}