{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/webqa-multihop-and-multimodal-qa","title":"WebQA: Multihop and Multimodal QA","arxiv_id":"2109.00590","date":"2021-09-01","proceeding":"CVPR 2022 1","authors":["Yingshan Chang","Mridu Narang","Hisami Suzuki","Guihong Cao","Jianfeng Gao","Yonatan Bisk"],"abstract":"Scaling Visual Question Answering (VQA) to the open-domain and multi-hop nature of web searches, requires fundamental advances in visual representation learning, knowledge aggregation, and language generation. In this work, we introduce WebQA, a challenging new benchmark that proves difficult for large-scale state-of-the-art models which lack language groundable visual representations for novel objects and the ability to reason, yet trivial for humans. WebQA mirrors the way humans use the web: 1) Ask a question, 2) Choose sources to aggregate, and 3) Produce a fluent language response. This is the behavior we should be expecting from IoT devices and digital assistants. Existing work prefers to assume that a model can either reason about knowledge in images or in text. WebQA includes a secondary text-only QA task to ensure improved visual performance does not come at the cost of language understanding. Our challenge for the community is to create unified multimodal reasoning models that answer questions regardless of the source modality, moving us closer to digital assistants that not only query language knowledge, but also the richer visual online world.","url_abs":"https://arxiv.org/abs/2109.00590v4","url_pdf":"https://arxiv.org/pdf/2109.00590v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"webqa-multihop-and-multimodal-qa","repo_url":"https://github.com/WebQnA/WebQA_Baseline","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"webqa-multihop-and-multimodal-qa","repo_url":"https://github.com/luomancs/remuq","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"webqa-multihop-and-multimodal-qa","repo_url":"https://github.com/shubham-gupta-iitr/mmmlX","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"image-retrieval","task_name":"Image Retrieval"},{"task_slug":"multimodal-reasoning","task_name":"Multimodal Reasoning"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"representation-learning","task_name":"Representation Learning"},{"task_slug":"text-generation","task_name":"Text Generation"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"},{"task_slug":"visual-question-answering","task_name":"Visual Question Answering (VQA)"}],"methods":[],"datasets_introduced":[{"slug":"webqa","name":"WebQA","full_name":""}],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2109.00590","atlas_url":"https://app.syntology.ai/?focus=2109.00590","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.00590"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/WebQnA/WebQA_Baseline","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/shubham-gupta-iitr/mmmlX","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/luomancs/remuq","reach":{"status":"ok"}}],"summary":{"ran_draft_wrong":2,"ran_honours":2,"ran":3,"unverified":6},"by_repo_kind":{"listed":{"samples":13,"ran":7,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":12,"samples":[{"code_sha256_prefix":"0f786c407fb1ee4c","entry":"swish","repo":"shubham-gupta-iitr/mmmlX","repo_kind":"listed","path":"pytorch_pretrained_bert/modeling.py","file_url":"https://github.com/shubham-gupta-iitr/mmmlX/blob/HEAD/pytorch_pretrained_bert/modeling.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":2,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"0f786c407fb1ee4c"}},{"code_sha256_prefix":"89ae895a1fff23b2","entry":"detectNum","repo":"WebQnA/WebQA_Baseline","repo_kind":"listed","path":"vlp/decode_webqa.py","file_url":"https://github.com/WebQnA/WebQA_Baseline/blob/HEAD/vlp/decode_webqa.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"89ae895a1fff23b2"}},{"code_sha256_prefix":"db8040436dc60177","entry":"gelu","repo":"shubham-gupta-iitr/mmmlX","repo_kind":"listed","path":"pytorch_pretrained_bert/modeling.py","file_url":"https://github.com/shubham-gupta-iitr/mmmlX/blob/HEAD/pytorch_pretrained_bert/modeling.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"db8040436dc60177"}},{"code_sha256_prefix":"232295eeb04a142e","entry":"load_vocab","repo":"shubham-gupta-iitr/mmmlX","repo_kind":"listed","path":"pytorch_pretrained_bert/tokenization.py","file_url":"https://github.com/shubham-gupta-iitr/mmmlX/blob/HEAD/pytorch_pretrained_bert/tokenization.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"232295eeb04a142e"}},{"code_sha256_prefix":"af64ec220e8bcdbc","entry":"url_to_filename","repo":"shubham-gupta-iitr/mmmlX","repo_kind":"listed","path":"pytorch_pretrained_bert/file_utils.py","file_url":"https://github.com/shubham-gupta-iitr/mmmlX/blob/HEAD/pytorch_pretrained_bert/file_utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"af64ec220e8bcdbc"}},{"code_sha256_prefix":"e7d542062316094a","entry":"warmup_constant","repo":"shubham-gupta-iitr/mmmlX","repo_kind":"listed","path":"pytorch_pretrained_bert/optimization.py","file_url":"https://github.com/shubham-gupta-iitr/mmmlX/blob/HEAD/pytorch_pretrained_bert/optimization.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"e7d542062316094a"}},{"code_sha256_prefix":"806f7e4b56361b23","entry":"warmup_linear","repo":"shubham-gupta-iitr/mmmlX","repo_kind":"listed","path":"pytorch_pretrained_bert/optimization.py","file_url":"https://github.com/shubham-gupta-iitr/mmmlX/blob/HEAD/pytorch_pretrained_bert/optimization.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"806f7e4b56361b23"}},{"code_sha256_prefix":"349d780dd4a89a37","entry":"cached_path","repo":"shubham-gupta-iitr/mmmlX","repo_kind":"listed","path":"pytorch_pretrained_bert/file_utils.py","file_url":"https://github.com/shubham-gupta-iitr/mmmlX/blob/HEAD/pytorch_pretrained_bert/file_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"349d780dd4a89a37"}},{"code_sha256_prefix":"db0ac56aaf6e35e6","entry":"filename_to_url","repo":"shubham-gupta-iitr/mmmlX","repo_kind":"listed","path":"pytorch_pretrained_bert/file_utils.py","file_url":"https://github.com/shubham-gupta-iitr/mmmlX/blob/HEAD/pytorch_pretrained_bert/file_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"db0ac56aaf6e35e6"}},{"code_sha256_prefix":"162a4118bd4b42b1","entry":"scatter_imbalance","repo":"shubham-gupta-iitr/mmmlX","repo_kind":"listed","path":"misc/data_parallel.py","file_url":"https://github.com/shubham-gupta-iitr/mmmlX/blob/HEAD/misc/data_parallel.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"162a4118bd4b42b1"}},{"code_sha256_prefix":"1f83e9eb8603cd57","entry":"scatter_kwargs_imbalance","repo":"shubham-gupta-iitr/mmmlX","repo_kind":"listed","path":"misc/data_parallel.py","file_url":"https://github.com/shubham-gupta-iitr/mmmlX/blob/HEAD/misc/data_parallel.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"1f83e9eb8603cd57"}},{"code_sha256_prefix":"35f7cddf90dd05d4","entry":"warmup_cosine","repo":"shubham-gupta-iitr/mmmlX","repo_kind":"listed","path":"pytorch_pretrained_bert/optimization.py","file_url":"https://github.com/shubham-gupta-iitr/mmmlX/blob/HEAD/pytorch_pretrained_bert/optimization.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"35f7cddf90dd05d4"}},{"code_sha256_prefix":"da7295883cd7da14","entry":"whitespace_tokenize","repo":"shubham-gupta-iitr/mmmlX","repo_kind":"listed","path":"pytorch_pretrained_bert/tokenization.py","file_url":"https://github.com/shubham-gupta-iitr/mmmlX/blob/HEAD/pytorch_pretrained_bert/tokenization.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"da7295883cd7da14"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}