{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/hybridqa-a-dataset-of-multi-hop-question","title":"HybridQA: A Dataset of Multi-Hop Question Answering over Tabular and Textual Data","arxiv_id":"2004.07347","date":"2020-04-15","proceeding":"Findings of the Association for Computational Linguistics 2020","authors":["Wenhu Chen","Hanwen Zha","Zhiyu Chen","Wenhan Xiong","Hong Wang","William Wang"],"abstract":"Existing question answering datasets focus on dealing with homogeneous information, based either only on text or KB/Table information alone. However, as human knowledge is distributed over heterogeneous forms, using homogeneous information alone might lead to severe coverage problems. To fill in the gap, we present HybridQA https://github.com/wenhuchen/HybridQA, a new large-scale question-answering dataset that requires reasoning on heterogeneous information. Each question is aligned with a Wikipedia table and multiple free-form corpora linked with the entities in the table. The questions are designed to aggregate both tabular information and text information, i.e., lack of either form would render the question unanswerable. We test with three different models: 1) a table-only model. 2) text-only model. 3) a hybrid model that combines heterogeneous information to find the answer. The experimental results show that the EM scores obtained by two baselines are below 20\\%, while the hybrid model can achieve an EM over 40\\%. This gap suggests the necessity to aggregate heterogeneous information in HybridQA. However, the hybrid model's score is still far behind human performance. Hence, HybridQA can serve as a challenging benchmark to study question answering with heterogeneous information.","url_abs":"https://arxiv.org/abs/2004.07347v3","url_pdf":"https://arxiv.org/pdf/2004.07347v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"hybridqa-a-dataset-of-multi-hop-question","repo_url":"https://github.com/wenhuchen/HybridQA","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"hybridqa-a-dataset-of-multi-hop-question","repo_url":"https://github.com/teacherpeterpan/Unsupervised-Multi-hop-QA","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"multi-hop-question-answering","task_name":"Multi-hop Question Answering"},{"task_slug":"question-answering","task_name":"Question Answering"}],"methods":[],"datasets_introduced":[{"slug":"hybridqa","name":"HybridQA","full_name":""}],"methods_introduced":[],"results":[{"leaderboard":"/sota/question-answering-on-hybridqa","task":"Question Answering","dataset":"HybridQA","model":"HYBRIDER","rank_in_archive_order":4,"of":4,"metrics":{"ANS-EM":"43.8"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2004.07347","atlas_url":"https://app.syntology.ai/?focus=2004.07347","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.07347"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/wenhuchen/HybridQA","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/teacherpeterpan/Unsupervised-Multi-hop-QA","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_honours":1,"ran":7,"ran_draft_wrong":3,"ran_violates":1,"unverified":3},"by_repo_kind":{"official":{"samples":3,"ran":3,"repositories":1},"listed":{"samples":10,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":2,"samples":[{"code_sha256_prefix":"9026d9da5d085bd4","entry":"compute_exact","repo":"wenhuchen/HybridQA","repo_kind":"official","path":"evaluate_script.py","file_url":"https://github.com/wenhuchen/HybridQA/blob/HEAD/evaluate_script.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9026d9da5d085bd4"}},{"code_sha256_prefix":"a745217943d329bd","entry":"get_bridge_entities","repo":"teacherpeterpan/Unsupervised-Multi-hop-QA","repo_kind":"listed","path":"MQA_QG/HybridQA/operations.py","file_url":"https://github.com/teacherpeterpan/Unsupervised-Multi-hop-QA/blob/HEAD/MQA_QG/HybridQA/operations.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a745217943d329bd"}},{"code_sha256_prefix":"aa6bc32d9c17a2f1","entry":"get_tokens","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"aa6bc32d9c17a2f1"}},{"code_sha256_prefix":"c7994a1f51a22564","entry":"include_bridge_entity","repo":"teacherpeterpan/Unsupervised-Multi-hop-QA","repo_kind":"listed","path":"MQA_QG/HotpotQA/operations.py","file_url":"https://github.com/teacherpeterpan/Unsupervised-Multi-hop-QA/blob/HEAD/MQA_QG/HotpotQA/operations.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c7994a1f51a22564"}},{"code_sha256_prefix":"86df5eac4985fe0e","entry":"isNaN","repo":"wenhuchen/HybridQA","repo_kind":"official","path":"train_stage12.py","file_url":"https://github.com/wenhuchen/HybridQA/blob/HEAD/train_stage12.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"86df5eac4985fe0e"}},{"code_sha256_prefix":"199bd77e59857d70","entry":"load_all_questions","repo":"teacherpeterpan/Unsupervised-Multi-hop-QA","repo_kind":"listed","path":"MQA_QG/run_ppl_ranking.py","file_url":"https://github.com/teacherpeterpan/Unsupervised-Multi-hop-QA/blob/HEAD/MQA_QG/run_ppl_ranking.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"199bd77e59857d70"}},{"code_sha256_prefix":"b8a1108847521ccb","entry":"load_hotpotQA","repo":"teacherpeterpan/Unsupervised-Multi-hop-QA","repo_kind":"listed","path":"prep_data_hotpotQA.py","file_url":"https://github.com/teacherpeterpan/Unsupervised-Multi-hop-QA/blob/HEAD/prep_data_hotpotQA.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b8a1108847521ccb"}},{"code_sha256_prefix":"6a96435eba311b08","entry":"normalize_answer","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"6a96435eba311b08"}},{"code_sha256_prefix":"fd256c34b37a0e87","entry":"phrase_match","repo":"teacherpeterpan/Unsupervised-Multi-hop-QA","repo_kind":"listed","path":"MQA_QG/HotpotQA/operations.py","file_url":"https://github.com/teacherpeterpan/Unsupervised-Multi-hop-QA/blob/HEAD/MQA_QG/HotpotQA/operations.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fd256c34b37a0e87"}},{"code_sha256_prefix":"8a1e9b4aa699789a","entry":"phrase_overlap","repo":"teacherpeterpan/Unsupervised-Multi-hop-QA","repo_kind":"listed","path":"MQA_QG/HybridQA/utils.py","file_url":"https://github.com/teacherpeterpan/Unsupervised-Multi-hop-QA/blob/HEAD/MQA_QG/HybridQA/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8a1e9b4aa699789a"}},{"code_sha256_prefix":"8f0d111c90b80c13","entry":"title_match","repo":"teacherpeterpan/Unsupervised-Multi-hop-QA","repo_kind":"listed","path":"prep_data_hotpotQA.py","file_url":"https://github.com/teacherpeterpan/Unsupervised-Multi-hop-QA/blob/HEAD/prep_data_hotpotQA.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8f0d111c90b80c13"}},{"code_sha256_prefix":"38c3f4fa8e089762","entry":"url2text","repo":"wenhuchen/HybridQA","repo_kind":"official","path":"train_stage12.py","file_url":"https://github.com/wenhuchen/HybridQA/blob/HEAD/train_stage12.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"38c3f4fa8e089762"}},{"code_sha256_prefix":"f1a4954ff85e0b97","entry":"construct_sample","repo":"teacherpeterpan/Unsupervised-Multi-hop-QA","repo_kind":"listed","path":"MQA_QG/run_multihop_generation.py","file_url":"https://github.com/teacherpeterpan/Unsupervised-Multi-hop-QA/blob/HEAD/MQA_QG/run_multihop_generation.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f1a4954ff85e0b97"}},{"code_sha256_prefix":"79bc6415d0faf9c6","entry":"convert_question_into_desc","repo":"teacherpeterpan/Unsupervised-Multi-hop-QA","repo_kind":"listed","path":"MQA_QG/HybridQA/operations.py","file_url":"https://github.com/teacherpeterpan/Unsupervised-Multi-hop-QA/blob/HEAD/MQA_QG/HybridQA/operations.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"79bc6415d0faf9c6"}},{"code_sha256_prefix":"04a978d51168a6db","entry":"get_passage","repo":"teacherpeterpan/Unsupervised-Multi-hop-QA","repo_kind":"listed","path":"MQA_QG/HybridQA/operations.py","file_url":"https://github.com/teacherpeterpan/Unsupervised-Multi-hop-QA/blob/HEAD/MQA_QG/HybridQA/operations.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"04a978d51168a6db"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}