{"url":"/method/vilbert","slug":"vilbert","name":"ViLBERT","full_name":"Vision-and-Language BERT","full_name_withheld":false,"description_markdown":"**Vision-and-Language BERT** (**ViLBERT**) is a [BERT](https://paperswithcode.com/method/bert)-based model for learning task-agnostic joint representations of image content and natural language. ViLBERT extend the popular BERT architecture to a multi-modal two-stream model, processing both visual and textual inputs in separate streams that interact through co-attentional [transformer](https://paperswithcode.com/method/transformer) layers.","description_state":"present","introduced_year":null,"introduced_by":{"title":"ViLBERT: Pretraining Task-Agnostic Visiolinguistic Representations for Vision-and-Language Tasks","paper":"/paper/vilbert-pretraining-task-agnostic","first_author":"Jiasen Lu","n_authors":4,"url_abs":null,"archive_paper_url":"https://paperswithcode.com/paper/vilbert-pretraining-task-agnostic"},"source":{"url":"https://arxiv.org/abs/1908.02265v1","title":"ViLBERT: Pretraining Task-Agnostic Visiolinguistic Representations for Vision-and-Language Tasks","url_on_a_paper_host":true},"code_snippet_url":null,"code_snippet_url_on_a_code_host":false,"categories":[{"area":"Computer Vision","area_id":"computer-vision","collection":"Vision and Language Pre-Trained Models","url":"/methods/category/vision-and-language-pre-trained-models","pwc_aliases":[]},{"area":"Natural Language Processing","area_id":"natural-language-processing","collection":"Transformers","url":"/methods/category/transformers","pwc_aliases":[]},{"area":"General","area_id":"general","collection":"Representation Learning","url":"/methods/category/representation-learning","pwc_aliases":[]}],"n_papers_tagged":30,"archive_num_papers":30,"papers_newest_first":[{"paper":null,"title":"A Multimodal Fusion Network For Student Emotion Recognition Based on Transformer and Tensor Product","date":"2024-03-13","arxiv_id":"2403.08511","n_code_links":0,"syntology":null},{"paper":"/paper/beyond-image-text-matching-verb-understanding","title":"Beyond Image-Text Matching: Verb Understanding in Multimodal Transformers Using Guided Masking","date":"2024-01-29","arxiv_id":"2401.16575","n_code_links":1,"syntology":null},{"paper":"/paper/the-bla-benchmark-investigating-basic","title":"The BLA Benchmark: Investigating Basic Language Abilities of Pre-Trained Multimodal Models","date":"2023-10-23","arxiv_id":"2310.15061","n_code_links":1,"syntology":{"ran":11,"of":12,"unverified":1,"pointer_only":0}},{"paper":"/paper/towards-a-performance-analysis-on-pre-trained","title":"Towards a performance analysis on pre-trained Visual Question Answering models for autonomous driving","date":"2023-07-18","arxiv_id":"2307.09329","n_code_links":1,"syntology":null},{"paper":null,"title":"Switch-BERT: Learning to Model Multimodal Interactions by Switching Attention and Input","date":"2023-06-25","arxiv_id":"2306.14182","n_code_links":0,"syntology":null},{"paper":null,"title":"Weakly Supervised Visual Question Answer Generation","date":"2023-06-11","arxiv_id":"2306.06622","n_code_links":0,"syntology":null},{"paper":"/paper/unified-multimodal-model-with-unlikelihood","title":"Unified Multimodal Model with Unlikelihood Training for Visual Dialog","date":"2022-11-23","arxiv_id":"2211.13235","n_code_links":1,"syntology":null},{"paper":null,"title":"A survey on knowledge-enhanced multimodal learning","date":"2022-11-19","arxiv_id":"2211.12328","n_code_links":0,"syntology":null},{"paper":null,"title":"Probing Cross-modal Semantics Alignment Capability from the Textual Perspective","date":"2022-10-18","arxiv_id":"2210.09550","n_code_links":0,"syntology":null},{"paper":"/paper/image-retrieval-from-contextual-descriptions-1","title":"Image Retrieval from Contextual Descriptions","date":"2022-03-29","arxiv_id":"2203.15867","n_code_links":1,"syntology":{"ran":2,"of":2,"unverified":0,"pointer_only":0}},{"paper":"/paper/tribert-human-centric-audio-visual","title":"TriBERT: Human-centric Audio-visual Representation Learning","date":"2021-12-01","arxiv_id":null,"n_code_links":1,"syntology":null},{"paper":null,"title":"Image Retrieval from Contextual Descriptions","date":"2021-11-16","arxiv_id":null,"n_code_links":0,"syntology":null},{"paper":null,"title":"Multimodal Learning: Are Captions All You Need?","date":"2021-11-16","arxiv_id":null,"n_code_links":0,"syntology":null},{"paper":"/paper/tribert-full-body-human-centric-audio-visual","title":"TriBERT: Full-body Human-centric Audio-visual Representation Learning for Visual Sound Separation","date":"2021-10-26","arxiv_id":"2110.13412","n_code_links":1,"syntology":{"ran":1,"of":2,"unverified":1,"pointer_only":2}},{"paper":"/paper/broaden-the-vision-geo-diverse-visual","title":"Broaden the Vision: Geo-Diverse Visual Commonsense Reasoning","date":"2021-09-14","arxiv_id":"2109.06860","n_code_links":1,"syntology":{"ran":0,"of":9,"unverified":9,"pointer_only":0}},{"paper":"/paper/enhance-multimodal-model-performance-with","title":"Enhance Multimodal Model Performance with Data Augmentation: Facebook Hateful Meme Challenge Solution","date":"2021-05-25","arxiv_id":"2105.13132","n_code_links":1,"syntology":null},{"paper":null,"title":"Chop Chop BERT: Visual Question Answering by Chopping VisualBERT's Heads","date":"2021-04-30","arxiv_id":"2104.14741","n_code_links":0,"syntology":null},{"paper":null,"title":"Playing Lottery Tickets with Vision and Language","date":"2021-04-23","arxiv_id":"2104.11832","n_code_links":0,"syntology":null},{"paper":"/paper/on-the-role-of-images-for-analyzing-claims-in","title":"On the Role of Images for Analyzing Claims in Social Media","date":"2021-03-17","arxiv_id":"2103.09602","n_code_links":1,"syntology":null},{"paper":"/paper/seeing-past-words-testing-the-cross-modal","title":"Seeing past words: Testing the cross-modal capabilities of pretrained V&L models on counting tasks","date":"2020-12-22","arxiv_id":"2012.12352","n_code_links":0,"syntology":null},{"paper":null,"title":"A Closer Look at the Robustness of Vision-and-Language Pre-trained Models","date":"2020-12-15","arxiv_id":"2012.08673","n_code_links":0,"syntology":null},{"paper":"/paper/a-multi-modal-method-for-satire-detection","title":"A Multi-Modal Method for Satire Detection using Textual and Visual Cues","date":"2020-10-13","arxiv_id":"2010.06671","n_code_links":1,"syntology":null},{"paper":"/paper/x-lxmert-paint-caption-and-answer-questions","title":"X-LXMERT: Paint, Caption and Answer Questions with Multi-Modal Transformers","date":"2020-09-23","arxiv_id":"2009.11278","n_code_links":2,"syntology":{"ran":1,"of":2,"unverified":1,"pointer_only":2}},{"paper":null,"title":"Contrastive Visual-Linguistic Pretraining","date":"2020-07-26","arxiv_id":"2007.13135","n_code_links":0,"syntology":null},{"paper":null,"title":"What Does BERT with Vision Look At?","date":"2020-07-01","arxiv_id":null,"n_code_links":0,"syntology":null},{"paper":null,"title":"Behind the Scene: Revealing the Secrets of Pre-trained Vision-and-Language Models","date":"2020-05-15","arxiv_id":"2005.07310","n_code_links":0,"syntology":null},{"paper":"/paper/words-aren-t-enough-their-order-matters-on","title":"Words aren't enough, their order matters: On the Robustness of Grounding Visual Referring Expressions","date":"2020-05-04","arxiv_id":"2005.01655","n_code_links":1,"syntology":null},{"paper":null,"title":"Generating Rationales in Visual Question Answering","date":"2020-04-04","arxiv_id":"2004.02032","n_code_links":0,"syntology":null},{"paper":"/paper/large-scale-pretraining-for-visual-dialog-a","title":"Large-scale Pretraining for Visual Dialog: A Simple State-of-the-Art Baseline","date":"2019-12-05","arxiv_id":"1912.02379","n_code_links":2,"syntology":{"ran":3,"of":6,"unverified":3,"pointer_only":0}},{"paper":"/paper/vilbert-pretraining-task-agnostic","title":"ViLBERT: Pretraining Task-Agnostic Visiolinguistic Representations for Vision-and-Language Tasks","date":"2019-08-06","arxiv_id":"1908.02265","n_code_links":11,"syntology":{"ran":10,"of":34,"unverified":24,"pointer_only":34}}],"papers_shown":30,"tasks":[{"task":"/task/visual-question-answering","name":"Visual Question Answering (VQA)","papers":9},{"task":"/task/visual-question-answering-1","name":"Visual Question Answering","papers":8},{"task":"/task/question-answering","name":"Question Answering","papers":7},{"task":"/task/retrieval","name":"Retrieval","papers":7},{"task":"/task/representation-learning","name":"Representation Learning","papers":6},{"task":"/task/image-captioning","name":"Image Captioning","papers":4},{"task":"/task/visual-commonsense-reasoning","name":"Visual Commonsense Reasoning","papers":4},{"task":"/task/image-retrieval","name":"Image Retrieval","papers":3},{"task":"/task/language-modelling","name":"Language Modelling","papers":3},{"task":"/task/referring-expression","name":"Referring Expression","papers":3},{"task":"/task/referring-expression-comprehension","name":"Referring Expression Comprehension","papers":3},{"task":"/task/visual-dialogue","name":"Visual Dialog","papers":3},{"task":"/task/visual-grounding","name":"Visual Grounding","papers":3},{"task":"/task/visual-reasoning","name":"Visual Reasoning","papers":3},{"task":"/task/answer-generation","name":"Answer Generation","papers":2},{"task":"/task/contrastive-learning","name":"Contrastive Learning","papers":2},{"task":"/task/image-text-retrieval","name":"Image-text Retrieval","papers":2},{"task":"/task/language-modeling","name":"Language Modeling","papers":2},{"task":"/task/multi-task-learning","name":"Multi-Task Learning","papers":2},{"task":"/task/pose-retrieval","name":"Pose Retrieval","papers":2}],"tasks_shown":20,"n_tasks":64,"usage_by_year":[{"year":"2019","papers":2},{"year":"2020","papers":9},{"year":"2021","papers":9},{"year":"2022","papers":4},{"year":"2023","papers":4},{"year":"2024","papers":2}],"row_source":"methods_table","archive":{"source":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","archive_url":"https://paperswithcode.com/method/vilbert"},"syntology_read_at":"2026-09-24T18:15:14+00:00"}