{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/bilinear-attention-networks","title":"Bilinear Attention Networks","arxiv_id":"1805.07932","date":"2018-05-21","proceeding":"NeurIPS 2018 12","authors":["Jin-Hwa Kim","Jaehyun Jun","Byoung-Tak Zhang"],"abstract":"Attention networks in multimodal learning provide an efficient way to utilize\ngiven visual information selectively. However, the computational cost to learn\nattention distributions for every pair of multimodal input channels is\nprohibitively expensive. To solve this problem, co-attention builds two\nseparate attention distributions for each modality neglecting the interaction\nbetween multimodal inputs. In this paper, we propose bilinear attention\nnetworks (BAN) that find bilinear attention distributions to utilize given\nvision-language information seamlessly. BAN considers bilinear interactions\namong two groups of input channels, while low-rank bilinear pooling extracts\nthe joint representations for each pair of channels. Furthermore, we propose a\nvariant of multimodal residual networks to exploit eight-attention maps of the\nBAN efficiently. We quantitatively and qualitatively evaluate our model on\nvisual question answering (VQA 2.0) and Flickr30k Entities datasets, showing\nthat BAN significantly outperforms previous methods and achieves new\nstate-of-the-arts on both datasets.","url_abs":"http://arxiv.org/abs/1805.07932v2","url_pdf":"http://arxiv.org/pdf/1805.07932v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"bilinear-attention-networks","repo_url":"https://github.com/jnhwkim/ban-vqa","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"bilinear-attention-networks","repo_url":"https://github.com/Cyanogenoid/vqa-counting","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"bilinear-attention-networks","repo_url":"https://github.com/ZephyrZhuQi/ssbaseline","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"bilinear-attention-networks","repo_url":"https://github.com/allenai/pythia","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"bilinear-attention-networks","repo_url":"https://github.com/facebookresearch/mmf","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"bilinear-attention-networks","repo_url":"https://github.com/facebookresearch/pythia","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"bilinear-attention-networks","repo_url":"https://github.com/jackroos/pythia","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"bilinear-attention-networks","repo_url":"https://github.com/ronghanghu/pythia","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}}],"tasks":[{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"},{"task_slug":"visual-question-answering","task_name":"Visual Question Answering (VQA)"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/phrase-grounding-on-flickr30k-entities-test","task":"Phrase Grounding","dataset":"Flickr30k Entities Test","model":"BAN (Bottom-Up detector)","rank_in_archive_order":11,"of":18,"metrics":{"R@1":"69.69","R@10":"86.35","R@5":"84.22"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-vqa-v2-test-dev","task":"Visual Question Answering (VQA)","dataset":"VQA v2 test-dev","model":"BAN+Glove+Counter","rank_in_archive_order":31,"of":56,"metrics":{"Accuracy":"70.04"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-vqa-v2-test-std","task":"Visual Question Answering (VQA)","dataset":"VQA v2 test-std","model":"BAN+Glove+Counter","rank_in_archive_order":27,"of":38,"metrics":{"overall":"70.4"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1805.07932","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1805.07932"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/mmf","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jnhwkim/ban-vqa","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/pythia","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ronghanghu/pythia","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Cyanogenoid/vqa-counting","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/allenai/pythia","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jackroos/pythia","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ZephyrZhuQi/ssbaseline","reach":{"status":"ok","spdx":"NOASSERTION"}}],"summary":{"ran_violates":3,"ran_honours":1,"unverified":9},"by_repo_kind":{"official":{"samples":7,"ran":3,"repositories":1},"listed":{"samples":6,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"cf2efa825f4afaf0","entry":"answer_filter","repo":"jnhwkim/ban-vqa","repo_kind":"official","path":"dataset.py","file_url":"https://github.com/jnhwkim/ban-vqa/blob/HEAD/dataset.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cf2efa825f4afaf0"}},{"code_sha256_prefix":"ac9f54e37ab206e4","entry":"is_howmany","repo":"jnhwkim/ban-vqa","repo_kind":"official","path":"dataset.py","file_url":"https://github.com/jnhwkim/ban-vqa/blob/HEAD/dataset.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ac9f54e37ab206e4"}},{"code_sha256_prefix":"1df9a5ffd9b38c34","entry":"pil_loader","repo":"jnhwkim/ban-vqa","repo_kind":"official","path":"utils.py","file_url":"https://github.com/jnhwkim/ban-vqa/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1df9a5ffd9b38c34"}},{"code_sha256_prefix":"b3299f7858d7d2f2","entry":"tile_2d_over_nd","repo":"Cyanogenoid/vqa-counting","repo_kind":"listed","path":"vqa-v2/model.py","file_url":"https://github.com/Cyanogenoid/vqa-counting/blob/HEAD/vqa-v2/model.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b3299f7858d7d2f2"}},{"code_sha256_prefix":"26e02979502211f1","entry":"apply_attention","repo":"Cyanogenoid/vqa-counting","repo_kind":"listed","path":"vqa-v2/model.py","file_url":"https://github.com/Cyanogenoid/vqa-counting/blob/HEAD/vqa-v2/model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"26e02979502211f1"}},{"code_sha256_prefix":"2c844ba6b5d8ece2","entry":"batch_accuracy","repo":"Cyanogenoid/vqa-counting","repo_kind":"listed","path":"vqa-v2/utils.py","file_url":"https://github.com/Cyanogenoid/vqa-counting/blob/HEAD/vqa-v2/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2c844ba6b5d8ece2"}},{"code_sha256_prefix":"28e89c10480659aa","entry":"collate_fn","repo":"Cyanogenoid/vqa-counting","repo_kind":"listed","path":"vqa-v2/data.py","file_url":"https://github.com/Cyanogenoid/vqa-counting/blob/HEAD/vqa-v2/data.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"28e89c10480659aa"}},{"code_sha256_prefix":"5a8370941ac95d70","entry":"compute_recall_with_logits","repo":"jnhwkim/ban-vqa","repo_kind":"official","path":"train_flickr.py","file_url":"https://github.com/jnhwkim/ban-vqa/blob/HEAD/train_flickr.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5a8370941ac95d70"}},{"code_sha256_prefix":"728c76c97b1bb97a","entry":"evaluate","repo":"jnhwkim/ban-vqa","repo_kind":"official","path":"train_flickr.py","file_url":"https://github.com/jnhwkim/ban-vqa/blob/HEAD/train_flickr.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"728c76c97b1bb97a"}},{"code_sha256_prefix":"8e41785566e403d2","entry":"extract_vocab","repo":"Cyanogenoid/vqa-counting","repo_kind":"listed","path":"vqa-v2/preprocess-vocab.py","file_url":"https://github.com/Cyanogenoid/vqa-counting/blob/HEAD/vqa-v2/preprocess-vocab.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8e41785566e403d2"}},{"code_sha256_prefix":"8c1252eacab71602","entry":"load_folder","repo":"jnhwkim/ban-vqa","repo_kind":"official","path":"utils.py","file_url":"https://github.com/jnhwkim/ban-vqa/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8c1252eacab71602"}},{"code_sha256_prefix":"bf90bd27fe08f3a2","entry":"load_imageid","repo":"jnhwkim/ban-vqa","repo_kind":"official","path":"utils.py","file_url":"https://github.com/jnhwkim/ban-vqa/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bf90bd27fe08f3a2"}},{"code_sha256_prefix":"1e3e251c3f07251d","entry":"prepare_answers","repo":"Cyanogenoid/vqa-counting","repo_kind":"listed","path":"vqa-v2/data.py","file_url":"https://github.com/Cyanogenoid/vqa-counting/blob/HEAD/vqa-v2/data.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1e3e251c3f07251d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}