{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/codeqa-a-question-answering-dataset-for","title":"CodeQA: A Question Answering Dataset for Source Code Comprehension","arxiv_id":"2109.08365","date":"2021-09-17","proceeding":"Findings (EMNLP) 2021 11","authors":["Chenxiao Liu","Xiaojun Wan"],"abstract":"We propose CodeQA, a free-form question answering dataset for the purpose of source code comprehension: given a code snippet and a question, a textual answer is required to be generated. CodeQA contains a Java dataset with 119,778 question-answer pairs and a Python dataset with 70,085 question-answer pairs. To obtain natural and faithful questions and answers, we implement syntactic rules and semantic analysis to transform code comments into question-answer pairs. We present the construction process and conduct systematic analysis of our dataset. Experiment results achieved by several neural baselines on our dataset are shown and discussed. While research on question-answering and machine reading comprehension develops rapidly, few prior work has drawn attention to code question answering. This new dataset can serve as a useful research benchmark for source code comprehension.","url_abs":"https://arxiv.org/abs/2109.08365v1","url_pdf":"https://arxiv.org/pdf/2109.08365v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"codeqa-a-question-answering-dataset-for","repo_url":"https://github.com/jadecxliu/codeqa","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"machine-reading-comprehension","task_name":"Machine Reading Comprehension"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"reading-comprehension","task_name":"Reading Comprehension"}],"methods":[],"datasets_introduced":[{"slug":"codeqa","name":"CodeQA","full_name":""}],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2109.08365","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2109.08365"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jadecxliu/codeqa","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":4,"unverified":7},"by_repo_kind":{"official":{"samples":11,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"835fca2953749b59","entry":"count_ngrams","repo":"jadecxliu/codeqa","repo_kind":"official","path":"codeBERT/code/bleu.py","file_url":"https://github.com/jadecxliu/codeqa/blob/HEAD/codeBERT/code/bleu.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"835fca2953749b59"}},{"code_sha256_prefix":"ba9348aac71a3f72","entry":"dec","repo":"jadecxliu/codeqa","repo_kind":"official","path":"codeBERT/code/eval/meteor/meteor.py","file_url":"https://github.com/jadecxliu/codeqa/blob/HEAD/codeBERT/code/eval/meteor/meteor.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ba9348aac71a3f72"}},{"code_sha256_prefix":"eaab50b5d9fea58a","entry":"enc","repo":"jadecxliu/codeqa","repo_kind":"official","path":"codeBERT/code/eval/meteor/meteor.py","file_url":"https://github.com/jadecxliu/codeqa/blob/HEAD/codeBERT/code/eval/meteor/meteor.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"eaab50b5d9fea58a"}},{"code_sha256_prefix":"ec4448373d72d636","entry":"precook","repo":"jadecxliu/codeqa","repo_kind":"official","path":"codeBERT/code/eval/bleu/bleu_scorer.py","file_url":"https://github.com/jadecxliu/codeqa/blob/HEAD/codeBERT/code/eval/bleu/bleu_scorer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ec4448373d72d636"}},{"code_sha256_prefix":"225b0921f040310c","entry":"compute_bleu","repo":"jadecxliu/codeqa","repo_kind":"official","path":"codeBERT/code/eval/bleu/google_bleu.py","file_url":"https://github.com/jadecxliu/codeqa/blob/HEAD/codeBERT/code/eval/bleu/google_bleu.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"225b0921f040310c"}},{"code_sha256_prefix":"7586e1d32f7325f7","entry":"cook_refs","repo":"jadecxliu/codeqa","repo_kind":"official","path":"codeBERT/code/eval/bleu/bleu_scorer.py","file_url":"https://github.com/jadecxliu/codeqa/blob/HEAD/codeBERT/code/eval/bleu/bleu_scorer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7586e1d32f7325f7"}},{"code_sha256_prefix":"019ae8bc6f98133f","entry":"cook_test","repo":"jadecxliu/codeqa","repo_kind":"official","path":"codeBERT/code/eval/bleu/bleu_scorer.py","file_url":"https://github.com/jadecxliu/codeqa/blob/HEAD/codeBERT/code/eval/bleu/bleu_scorer.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"019ae8bc6f98133f"}},{"code_sha256_prefix":"ecccdfbec38d40e7","entry":"corpus_bleu","repo":"jadecxliu/codeqa","repo_kind":"official","path":"codeBERT/code/eval/bleu/google_bleu.py","file_url":"https://github.com/jadecxliu/codeqa/blob/HEAD/codeBERT/code/eval/bleu/google_bleu.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ecccdfbec38d40e7"}},{"code_sha256_prefix":"8e177567218199c4","entry":"my_lcs","repo":"jadecxliu/codeqa","repo_kind":"official","path":"codeBERT/code/eval/rouge/rouge.py","file_url":"https://github.com/jadecxliu/codeqa/blob/HEAD/codeBERT/code/eval/rouge/rouge.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8e177567218199c4"}},{"code_sha256_prefix":"c3b65b5384d8541c","entry":"nltk_corpus_bleu","repo":"jadecxliu/codeqa","repo_kind":"official","path":"codeBERT/code/eval/bleu/nltk_bleu.py","file_url":"https://github.com/jadecxliu/codeqa/blob/HEAD/codeBERT/code/eval/bleu/nltk_bleu.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c3b65b5384d8541c"}},{"code_sha256_prefix":"bb85d1a914a54c97","entry":"nltk_sentence_bleu","repo":"jadecxliu/codeqa","repo_kind":"official","path":"codeBERT/code/eval/bleu/nltk_bleu.py","file_url":"https://github.com/jadecxliu/codeqa/blob/HEAD/codeBERT/code/eval/bleu/nltk_bleu.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bb85d1a914a54c97"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}