{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/multimodal-compact-bilinear-pooling-for","title":"Multimodal Compact Bilinear Pooling for Visual Question Answering and Visual Grounding","arxiv_id":"1606.01847","date":"2016-06-06","proceeding":"EMNLP 2016 11","authors":["Akira Fukui","Dong Huk Park","Daylen Yang","Anna Rohrbach","Trevor Darrell","Marcus Rohrbach"],"abstract":"Modeling textual or visual information with vector representations trained\nfrom large language or visual datasets has been successfully explored in recent\nyears. However, tasks such as visual question answering require combining these\nvector representations with each other. Approaches to multimodal pooling\ninclude element-wise product or sum, as well as concatenation of the visual and\ntextual representations. We hypothesize that these methods are not as\nexpressive as an outer product of the visual and textual vectors. As the outer\nproduct is typically infeasible due to its high dimensionality, we instead\npropose utilizing Multimodal Compact Bilinear pooling (MCB) to efficiently and\nexpressively combine multimodal features. We extensively evaluate MCB on the\nvisual question answering and grounding tasks. We consistently show the benefit\nof MCB over ablations without MCB. For visual question answering, we present an\narchitecture which uses MCB twice, once for predicting attention over spatial\nfeatures and again to combine the attended representation with the question\nrepresentation. This model outperforms the state-of-the-art on the Visual7W\ndataset and the VQA challenge.","url_abs":"http://arxiv.org/abs/1606.01847v3","url_pdf":"http://arxiv.org/pdf/1606.01847v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"multimodal-compact-bilinear-pooling-for","repo_url":"https://github.com/akirafukui/vqa-mcb","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"caffe2","reach":{"status":"ok","spdx":"BSD-2-Clause"}},{"paper_slug":"multimodal-compact-bilinear-pooling-for","repo_url":"https://github.com/Adam1679/mutan-article-net","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"unanswered"}},{"paper_slug":"multimodal-compact-bilinear-pooling-for","repo_url":"https://github.com/Cadene/vqa.pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"unanswered"}},{"paper_slug":"multimodal-compact-bilinear-pooling-for","repo_url":"https://github.com/JoonSeongPark/vqa","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"multimodal-compact-bilinear-pooling-for","repo_url":"https://github.com/MarcBS/keras","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":null},{"paper_slug":"multimodal-compact-bilinear-pooling-for","repo_url":"https://github.com/arunmallya/simple-vqa","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"torch","reach":{"status":"ok"}},{"paper_slug":"multimodal-compact-bilinear-pooling-for","repo_url":"https://github.com/gabegrand/adversarial-vqa","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"multimodal-compact-bilinear-pooling-for","repo_url":"https://github.com/jnhwkim/cbp","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"torch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"multimodal-compact-bilinear-pooling-for","repo_url":"https://github.com/vuhoangminh/vqa_medical","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"unanswered"}},{"paper_slug":"multimodal-compact-bilinear-pooling-for","repo_url":"https://github.com/yikang-li/iqan","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"unanswered"}}],"tasks":[{"task_slug":"phrase-grounding","task_name":"Phrase Grounding"},{"task_slug":"visual-grounding","task_name":"Visual Grounding"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"},{"task_slug":"visual-question-answering","task_name":"Visual Question Answering (VQA)"}],"methods":[{"method_slug":"1x1-convolution","method_name":"1x1 Convolution"},{"method_slug":"average-pooling","method_name":"Average Pooling"},{"method_slug":"batch-normalization","method_name":"Batch Normalization"},{"method_slug":"bottleneck-residual-block","method_name":"Bottleneck Residual Block"},{"method_slug":"convolution","method_name":"Convolution"},{"method_slug":"global-average-pooling","method_name":"Global Average Pooling"},{"method_slug":"kaiming-initialization","method_name":"Kaiming Initialization"},{"method_slug":"max-pooling","method_name":"Max Pooling"},{"method_slug":"relu","method_name":"ReLU"},{"method_slug":"residual-block","method_name":"Residual Block"},{"method_slug":"residual-connection","method_name":"Residual Connection"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/phrase-grounding-on-flickr30k-entities-test","task":"Phrase Grounding","dataset":"Flickr30k Entities Test","model":"MCB","rank_in_archive_order":12,"of":18,"metrics":{"R@1":"48.69"},"uses_additional_data":false},{"leaderboard":"/sota/phrase-grounding-on-referit","task":"Phrase Grounding","dataset":"ReferIt","model":"MCB","rank_in_archive_order":3,"of":3,"metrics":{"Accuracy":"28.91"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-coco-visual-1","task":"Visual Question Answering (VQA)","dataset":"COCO Visual Question Answering (VQA) real images 1.0 multiple choice","model":"MCB 7 att.","rank_in_archive_order":1,"of":10,"metrics":{"Percentage correct":"70.1"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-coco-visual-4","task":"Visual Question Answering (VQA)","dataset":"COCO Visual Question Answering (VQA) real images 1.0 open ended","model":"MCB 7 att.","rank_in_archive_order":1,"of":14,"metrics":{"Percentage correct":"66.5"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-vqa-v1-test-dev","task":"Visual Question Answering (VQA)","dataset":"VQA v1 test-dev","model":"MCB (ResNet)","rank_in_archive_order":3,"of":7,"metrics":{"Accuracy":"64.2"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-vqa-v2-test-dev","task":"Visual Question Answering (VQA)","dataset":"VQA v2 test-dev","model":"MCB","rank_in_archive_order":45,"of":56,"metrics":{"Accuracy":"64.7"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-visual7w","task":"Visual Question Answering (VQA)","dataset":"Visual7W","model":"MCB+Att.","rank_in_archive_order":4,"of":4,"metrics":{"Percentage correct":"62.2"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1606.01847","mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}