{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/hotpotqa-a-dataset-for-diverse-explainable","title":"HotpotQA: A Dataset for Diverse, Explainable Multi-hop Question Answering","arxiv_id":"1809.09600","date":"2018-09-25","proceeding":"EMNLP 2018 10","authors":["Zhilin Yang","Peng Qi","Saizheng Zhang","Yoshua Bengio","William W. Cohen","Ruslan Salakhutdinov","Christopher D. Manning"],"abstract":"Existing question answering (QA) datasets fail to train QA systems to perform\ncomplex reasoning and provide explanations for answers. We introduce HotpotQA,\na new dataset with 113k Wikipedia-based question-answer pairs with four key\nfeatures: (1) the questions require finding and reasoning over multiple\nsupporting documents to answer; (2) the questions are diverse and not\nconstrained to any pre-existing knowledge bases or knowledge schemas; (3) we\nprovide sentence-level supporting facts required for reasoning, allowing QA\nsystems to reason with strong supervision and explain the predictions; (4) we\noffer a new type of factoid comparison questions to test QA systems' ability to\nextract relevant facts and perform necessary comparison. We show that HotpotQA\nis challenging for the latest QA systems, and the supporting facts enable\nmodels to improve performance and make explainable predictions.","url_abs":"http://arxiv.org/abs/1809.09600v1","url_pdf":"http://arxiv.org/pdf/1809.09600v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"hotpotqa-a-dataset-for-diverse-explainable","repo_url":"https://github.com/hotpotqa/hotpot","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"hotpotqa-a-dataset-for-diverse-explainable","repo_url":"https://github.com/mangopy/direct-rag-learning","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"multi-hop-question-answering","task_name":"Multi-hop Question Answering"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"sentence","task_name":"Sentence"}],"methods":[],"datasets_introduced":[{"slug":"hotpotqa","name":"HotpotQA","full_name":""}],"methods_introduced":[],"results":[{"leaderboard":"/sota/question-answering-on-hotpotqa","task":"Question Answering","dataset":"HotpotQA","model":"SAFSR model","rank_in_archive_order":34,"of":72,"metrics":{"ANS-EM":"0.589","ANS-F1":"0.716","JOINT-EM":"0.345","JOINT-F1":"0.598","SUP-EM":"0.480","SUP-F1":"0.757"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-hotpotqa","task":"Question Answering","dataset":"HotpotQA","model":"Baseline Model","rank_in_archive_order":64,"of":72,"metrics":{"ANS-EM":"0.240","ANS-F1":"0.329","JOINT-EM":"0.019","JOINT-F1":"0.162","SUP-EM":"0.039","SUP-F1":"0.377"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/1809.09600","atlas_url":"https://app.syntology.ai/?focus=1809.09600","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1809.09600"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mangopy/direct-rag-learning","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/hotpotqa/hotpot","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_violates":1,"ran_honours":1,"ran_draft_wrong":1,"unverified":5},"by_repo_kind":{"official":{"samples":8,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"9d0dc82a4491f803","entry":"exact_match_score","repo":"hotpotqa/hotpot","repo_kind":"official","path":"hotpot_evaluate_v1.py","file_url":"https://github.com/hotpotqa/hotpot/blob/HEAD/hotpot_evaluate_v1.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9d0dc82a4491f803"}},{"code_sha256_prefix":"32a5f1733d9a8971","entry":"f1_score","repo":"hotpotqa/hotpot","repo_kind":"official","path":"hotpot_evaluate_v1.py","file_url":"https://github.com/hotpotqa/hotpot/blob/HEAD/hotpot_evaluate_v1.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"32a5f1733d9a8971"}},{"code_sha256_prefix":"dae7ab386661a4f4","entry":"normalize_answer","repo":"hotpotqa/hotpot","repo_kind":"official","path":"hotpot_evaluate_v1.py","file_url":"https://github.com/hotpotqa/hotpot/blob/HEAD/hotpot_evaluate_v1.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"dae7ab386661a4f4"}},{"code_sha256_prefix":"5733fa44ca6deb1f","entry":"find_nearest","repo":"hotpotqa/hotpot","repo_kind":"official","path":"prepro.py","file_url":"https://github.com/hotpotqa/hotpot/blob/HEAD/prepro.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5733fa44ca6deb1f"}},{"code_sha256_prefix":"40e9cb6c8ca7fb3d","entry":"fix_span","repo":"hotpotqa/hotpot","repo_kind":"official","path":"prepro.py","file_url":"https://github.com/hotpotqa/hotpot/blob/HEAD/prepro.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"40e9cb6c8ca7fb3d"}},{"code_sha256_prefix":"ffc8a36cce91c106","entry":"get_buckets","repo":"hotpotqa/hotpot","repo_kind":"official","path":"util.py","file_url":"https://github.com/hotpotqa/hotpot/blob/HEAD/util.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ffc8a36cce91c106"}},{"code_sha256_prefix":"2be785f92fd075df","entry":"has_digit","repo":"hotpotqa/hotpot","repo_kind":"official","path":"util.py","file_url":"https://github.com/hotpotqa/hotpot/blob/HEAD/util.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2be785f92fd075df"}},{"code_sha256_prefix":"c15e3bee3bb007b7","entry":"prepro","repo":"hotpotqa/hotpot","repo_kind":"official","path":"util.py","file_url":"https://github.com/hotpotqa/hotpot/blob/HEAD/util.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c15e3bee3bb007b7"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}