{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/coqa-a-conversational-question-answering","title":"CoQA: A Conversational Question Answering Challenge","arxiv_id":"1808.07042","date":"2018-08-21","proceeding":"TACL 2019 3","authors":["Siva Reddy","Danqi Chen","Christopher D. Manning"],"abstract":"Humans gather information by engaging in conversations involving a series of\ninterconnected questions and answers. For machines to assist in information\ngathering, it is therefore essential to enable them to answer conversational\nquestions. We introduce CoQA, a novel dataset for building Conversational\nQuestion Answering systems. Our dataset contains 127k questions with answers,\nobtained from 8k conversations about text passages from seven diverse domains.\nThe questions are conversational, and the answers are free-form text with their\ncorresponding evidence highlighted in the passage. We analyze CoQA in depth and\nshow that conversational questions have challenging phenomena not present in\nexisting reading comprehension datasets, e.g., coreference and pragmatic\nreasoning. We evaluate strong conversational and reading comprehension models\non CoQA. The best system obtains an F1 score of 65.4%, which is 23.4 points\nbehind human performance (88.8%), indicating there is ample room for\nimprovement. We launch CoQA as a challenge to the community at\nhttp://stanfordnlp.github.io/coqa/","url_abs":"http://arxiv.org/abs/1808.07042v2","url_pdf":"http://arxiv.org/pdf/1808.07042v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"coqa-a-conversational-question-answering","repo_url":"https://github.com/iit-nlp-research/chatgpt-crawler","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"coqa-a-conversational-question-answering","repo_url":"https://github.com/leozhoujf/DataSciComp","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"paddle","reach":{"status":"unanswered"}},{"paper_slug":"coqa-a-conversational-question-answering","repo_url":"https://github.com/mrzjy/sunburst","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"unanswered"}},{"paper_slug":"coqa-a-conversational-question-answering","repo_url":"https://github.com/stanfordnlp/coqa-baselines","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":null,"task_name":"8k"},{"task_slug":"conversational-question-answering","task_name":"Conversational Question Answering"},{"task_slug":"generative-question-answering","task_name":"Generative Question Answering"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"reading-comprehension","task_name":"Reading Comprehension"}],"methods":[],"datasets_introduced":[{"slug":"coqa","name":"CoQA","full_name":"Conversational Question Answering Challenge"}],"methods_introduced":[],"results":[{"leaderboard":"/sota/generative-question-answering-on-coqa","task":"Generative Question Answering","dataset":"CoQA","model":"PGNet","rank_in_archive_order":3,"of":3,"metrics":{"F1-Score":"45.4"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-coqa","task":"Question Answering","dataset":"CoQA","model":"DrQA + seq2seq with copy attention (single model)","rank_in_archive_order":4,"of":9,"metrics":{"In-domain":"67.0","Out-of-domain":"60.4","Overall":"65.1"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-coqa","task":"Question Answering","dataset":"CoQA","model":"Vanilla DrQA (single model)","rank_in_archive_order":5,"of":9,"metrics":{"In-domain":"54.5","Out-of-domain":"47.9","Overall":"52.6"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1808.07042","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1808.07042"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/stanfordnlp/coqa-baselines","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/iit-nlp-research/chatgpt-crawler","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/leozhoujf/DataSciComp","reach":{"status":"unanswered"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mrzjy/sunburst","reach":{"status":"unanswered"}}],"summary":{"ran_draft_wrong":2},"by_repo_kind":{"listed":{"samples":2,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"7b38a76efd8dcf6b","entry":"get_str","repo":"stanfordnlp/coqa-baselines","repo_kind":"listed","path":"scripts/gen_pipeline_data.py","file_url":"https://github.com/stanfordnlp/coqa-baselines/blob/HEAD/scripts/gen_pipeline_data.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7b38a76efd8dcf6b"}},{"code_sha256_prefix":"a2fdbbe11e2cf910","entry":"normalize_answer","repo":"stanfordnlp/coqa-baselines","repo_kind":"listed","path":"scripts/gen_pipeline_data.py","file_url":"https://github.com/stanfordnlp/coqa-baselines/blob/HEAD/scripts/gen_pipeline_data.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a2fdbbe11e2cf910"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}