{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/conditionalqa-a-complex-reading-comprehension","title":"ConditionalQA: A Complex Reading Comprehension Dataset with Conditional Answers","arxiv_id":"2110.06884","date":"2021-10-13","proceeding":"ACL 2022 5","authors":["Haitian Sun","William W. Cohen","Ruslan Salakhutdinov"],"abstract":"We describe a Question Answering (QA) dataset that contains complex questions with conditional answers, i.e. the answers are only applicable when certain conditions apply. We call this dataset ConditionalQA. In addition to conditional answers, the dataset also features: (1) long context documents with information that is related in logically complex ways; (2) multi-hop questions that require compositional logical reasoning; (3) a combination of extractive questions, yes/no questions, questions with multiple answers, and not-answerable questions; (4) questions asked without knowing the answers. We show that ConditionalQA is challenging for many of the existing QA models, especially in selecting answer conditions. We believe that this dataset will motivate further research in answering complex questions over long documents. Data and leaderboard are publicly available at \\url{https://github.com/haitian-sun/ConditionalQA}.","url_abs":"https://arxiv.org/abs/2110.06884v1","url_pdf":"https://arxiv.org/pdf/2110.06884v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"conditionalqa-a-complex-reading-comprehension","repo_url":"https://github.com/haitian-sun/conditionalqa","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"none","reach":null},{"paper_slug":"conditionalqa-a-complex-reading-comprehension","repo_url":"https://github.com/dmis-lab/seqtagqa","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"logical-reasoning","task_name":"Logical Reasoning"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"reading-comprehension","task_name":"Reading Comprehension"}],"methods":[],"datasets_introduced":[{"slug":"conditionalqa","name":"ConditionalQA","full_name":""}],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2110.06884","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.06884"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/dmis-lab/seqtagqa","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/haitian-sun/conditionalqa","reach":null}],"summary":{"ran_honours":1,"ran_draft_wrong":1,"unverified":1},"by_repo_kind":{"official":{"samples":3,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":3,"samples":[{"code_sha256_prefix":"4f18a3f6971da6c1","entry":"compute_metrics","repo":"haitian-sun/conditionalqa","repo_kind":"official","path":"evaluate.py","file_url":"https://github.com/haitian-sun/conditionalqa/blob/HEAD/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4f18a3f6971da6c1"}},{"code_sha256_prefix":"932e948eecf1f227","entry":"load_answers","repo":"haitian-sun/conditionalqa","repo_kind":"official","path":"evaluate.py","file_url":"https://github.com/haitian-sun/conditionalqa/blob/HEAD/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"932e948eecf1f227"}},{"code_sha256_prefix":"1d52d2caf114bd51","entry":"evaluate","repo":"haitian-sun/conditionalqa","repo_kind":"official","path":"evaluate.py","file_url":"https://github.com/haitian-sun/conditionalqa/blob/HEAD/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1d52d2caf114bd51"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}