{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/weakly-supervised-visual-retriever-reader-for","title":"Weakly-Supervised Visual-Retriever-Reader for Knowledge-based Question Answering","arxiv_id":"2109.04014","date":"2021-09-09","proceeding":"EMNLP 2021 11","authors":["Man Luo","Yankai Zeng","Pratyay Banerjee","Chitta Baral"],"abstract":"Knowledge-based visual question answering (VQA) requires answering questions with external knowledge in addition to the content of images. One dataset that is mostly used in evaluating knowledge-based VQA is OK-VQA, but it lacks a gold standard knowledge corpus for retrieval. Existing work leverage different knowledge bases (e.g., ConceptNet and Wikipedia) to obtain external knowledge. Because of varying knowledge bases, it is hard to fairly compare models' performance. To address this issue, we collect a natural language knowledge base that can be used for any VQA system. Moreover, we propose a Visual Retriever-Reader pipeline to approach knowledge-based VQA. The visual retriever aims to retrieve relevant knowledge, and the visual reader seeks to predict answers based on given knowledge. We introduce various ways to retrieve knowledge using text and images and two reader styles: classification and extraction. Both the retriever and reader are trained with weak supervision. Our experimental results show that a good retriever can significantly improve the reader's performance on the OK-VQA challenge. The code and corpus are provided in https://github.com/luomancs/retriever\\_reader\\_for\\_okvqa.git","url_abs":"https://arxiv.org/abs/2109.04014v1","url_pdf":"https://arxiv.org/pdf/2109.04014v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"weakly-supervised-visual-retriever-reader-for","repo_url":"https://github.com/luomancs/retriever_reader_for_okvqa","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":null},{"paper_slug":"weakly-supervised-visual-retriever-reader-for","repo_url":"https://github.com/xinwei666/mmgenerativeir","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"},{"task_slug":"visual-question-answering","task_name":"Visual Question Answering (VQA)"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2109.04014","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.04014"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/xinwei666/mmgenerativeir","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/luomancs/retriever_reader_for_okvqa","reach":null}],"summary":{"ran":1,"ran_draft_wrong":1,"ran_fixture":1,"unverified":1},"by_repo_kind":{"official":{"samples":3,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":1,"samples":[{"code_sha256_prefix":"30c98cb903350faa","entry":"Reader","repo":"luomancs/retriever_reader_for_okvqa","repo_kind":"official","path":"DPR/dpr/model_lxmert/reader.py","file_url":"https://github.com/luomancs/retriever_reader_for_okvqa/blob/HEAD/DPR/dpr/model_lxmert/reader.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"30c98cb903350faa"}},{"code_sha256_prefix":"fcb2544f4ef7c452","entry":"_calc_mml","repo":"luomancs/retriever_reader_for_okvqa","repo_kind":"official","path":"DPR/dpr/model_lxmert/reader.py","file_url":"https://github.com/luomancs/retriever_reader_for_okvqa/blob/HEAD/DPR/dpr/model_lxmert/reader.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fcb2544f4ef7c452"}},{"code_sha256_prefix":"1df81ea9d2da7b70","entry":"compute_loss","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"1df81ea9d2da7b70"}},{"code_sha256_prefix":"62e3d8ee4f0d3ff3","entry":"init_weights","repo":"luomancs/retriever_reader_for_okvqa","repo_kind":"official","path":"DPR/dpr/model_lxmert/reader.py","file_url":"https://github.com/luomancs/retriever_reader_for_okvqa/blob/HEAD/DPR/dpr/model_lxmert/reader.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"62e3d8ee4f0d3ff3"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}