{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/plug-and-play-vqa-zero-shot-vqa-by-conjoining","title":"Plug-and-Play VQA: Zero-shot VQA by Conjoining Large Pretrained Models with Zero Training","arxiv_id":"2210.08773","date":"2022-10-17","proceeding":null,"authors":["Anthony Meng Huat Tiong","Junnan Li","Boyang Li","Silvio Savarese","Steven C. H. Hoi"],"abstract":"Visual question answering (VQA) is a hallmark of vision and language reasoning and a challenging task under the zero-shot setting. We propose Plug-and-Play VQA (PNP-VQA), a modular framework for zero-shot VQA. In contrast to most existing works, which require substantial adaptation of pretrained language models (PLMs) for the vision modality, PNP-VQA requires no additional training of the PLMs. Instead, we propose to use natural language and network interpretation as an intermediate representation that glues pretrained models together. We first generate question-guided informative image captions, and pass the captions to a PLM as context for question answering. Surpassing end-to-end trained baselines, PNP-VQA achieves state-of-the-art results on zero-shot VQAv2 and GQA. With 11B parameters, it outperforms the 80B-parameter Flamingo model by 8.5% on VQAv2. With 738M PLM parameters, PNP-VQA achieves an improvement of 9.1% on GQA over FewVLM with 740M PLM parameters. Code is released at https://github.com/salesforce/LAVIS/tree/main/projects/pnp-vqa","url_abs":"https://arxiv.org/abs/2210.08773v3","url_pdf":"https://arxiv.org/pdf/2210.08773v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"plug-and-play-vqa-zero-shot-vqa-by-conjoining","repo_url":"https://github.com/salesforce/lavis","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"BSD-3-Clause"}},{"paper_slug":"plug-and-play-vqa-zero-shot-vqa-by-conjoining","repo_url":"https://github.com/Tzoulio/Large_Models_Dialogue_for_Active_Perception","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"plug-and-play-vqa-zero-shot-vqa-by-conjoining","repo_url":"https://github.com/abril4416/kgen_vqa","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"image-captioning","task_name":"Image Captioning"},{"task_slug":"network-interpretation","task_name":"Network Interpretation"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"},{"task_slug":"visual-question-answering","task_name":"Visual Question Answering (VQA)"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/visual-question-answering-on-gqa-test-dev","task":"Visual Question Answering (VQA)","dataset":"GQA test-dev","model":"PNP-VQA","rank_in_archive_order":13,"of":17,"metrics":{"Accuracy":"41.9"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-ok-vqa","task":"Visual Question Answering (VQA)","dataset":"OK-VQA","model":"PNP-VQA","rank_in_archive_order":31,"of":37,"metrics":{"Accuracy":"35.9"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-vqa-v2-test-dev","task":"Visual Question Answering (VQA)","dataset":"VQA v2 test-dev","model":"PNP-VQA","rank_in_archive_order":44,"of":56,"metrics":{"Accuracy":"64.8"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-on-vqa-v2-val","task":"Visual Question Answering (VQA)","dataset":"VQA v2 val","model":"PNP-VQA","rank_in_archive_order":2,"of":11,"metrics":{"Accuracy":"63.3"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2210.08773","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.08773"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/salesforce/lavis","reach":{"status":"ok","spdx":"BSD-3-Clause"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/abril4416/kgen_vqa","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Tzoulio/Large_Models_Dialogue_for_Active_Perception","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":1,"ran":1,"unverified":1},"by_repo_kind":{"listed":{"samples":3,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":3,"samples":[{"code_sha256_prefix":"c6ec173f19f5c34d","entry":"interpolate_pos_embed","repo":"Tzoulio/Large_Models_Dialogue_for_Active_Perception","repo_kind":"listed","path":"llm-vqa_dialogue/lavis/models/vit.py","file_url":"https://github.com/Tzoulio/Large_Models_Dialogue_for_Active_Perception/blob/HEAD/llm-vqa_dialogue/lavis/models/vit.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"c6ec173f19f5c34d"}},{"code_sha256_prefix":"cb33571427334815","entry":"tile","repo":"Tzoulio/Large_Models_Dialogue_for_Active_Perception","repo_kind":"listed","path":"llm-vqa_dialogue/lavis/models/base_model.py","file_url":"https://github.com/Tzoulio/Large_Models_Dialogue_for_Active_Perception/blob/HEAD/llm-vqa_dialogue/lavis/models/base_model.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"cb33571427334815"}},{"code_sha256_prefix":"0ec9fc2025c16f65","entry":"all_gather_with_grad","repo":"Tzoulio/Large_Models_Dialogue_for_Active_Perception","repo_kind":"listed","path":"llm-vqa_dialogue/lavis/models/base_model.py","file_url":"https://github.com/Tzoulio/Large_Models_Dialogue_for_Active_Perception/blob/HEAD/llm-vqa_dialogue/lavis/models/base_model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"0ec9fc2025c16f65"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}