{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/codexglue-a-machine-learning-benchmark","title":"CodeXGLUE: A Machine Learning Benchmark Dataset for Code Understanding and Generation","arxiv_id":"2102.04664","date":"2021-02-09","proceeding":null,"authors":["Shuai Lu","Daya Guo","Shuo Ren","JunJie Huang","Alexey Svyatkovskiy","Ambrosio Blanco","Colin Clement","Dawn Drain","Daxin Jiang","Duyu Tang","Ge Li","Lidong Zhou","Linjun Shou","Long Zhou","Michele Tufano","Ming Gong","Ming Zhou","Nan Duan","Neel Sundaresan","Shao Kun Deng","Shengyu Fu","Shujie Liu"],"abstract":"Benchmark datasets have a significant impact on accelerating research in programming language tasks. In this paper, we introduce CodeXGLUE, a benchmark dataset to foster machine learning research for program understanding and generation. CodeXGLUE includes a collection of 10 tasks across 14 datasets and a platform for model evaluation and comparison. CodeXGLUE also features three baseline systems, including the BERT-style, GPT-style, and Encoder-Decoder models, to make it easy for researchers to use the platform. The availability of such data and baselines can help the development and validation of new methods that can be applied to various program understanding and generation problems.","url_abs":"https://arxiv.org/abs/2102.04664v2","url_pdf":"https://arxiv.org/pdf/2102.04664v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"codexglue-a-machine-learning-benchmark","repo_url":"https://github.com/microsoft/CodeXGLUE","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"codexglue-a-machine-learning-benchmark","repo_url":"https://github.com/Avmb/semantic_neq_game","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"unanswered"}},{"paper_slug":"codexglue-a-machine-learning-benchmark","repo_url":"https://github.com/deeplearnxmu/unigencoder","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"codexglue-a-machine-learning-benchmark","repo_url":"https://github.com/facebookresearch/CodeGen","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"codexglue-a-machine-learning-benchmark","repo_url":"https://github.com/kilimanj4r0/code-summarization-beyond-function-level","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"codexglue-a-machine-learning-benchmark","repo_url":"https://github.com/sberbank-ai/fusion_brain_aij2021","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"unanswered"}},{"paper_slug":"codexglue-a-machine-learning-benchmark","repo_url":"https://github.com/yueyuel/programgen-lms-reliability","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"unanswered"}}],"tasks":[{"task_slug":"machine-learning","task_name":"BIG-bench Machine Learning"},{"task_slug":"clone-detection","task_name":"Clone Detection"},{"task_slug":"cloze-test","task_name":"Cloze Test"},{"task_slug":"code-completion","task_name":"Code Completion"},{"task_slug":"code-generation","task_name":"Code Generation"},{"task_slug":"code-repair","task_name":"Code Repair"},{"task_slug":"code-search","task_name":"Code Search"},{"task_slug":"code-summarization-1","task_name":"Code Summarization"},{"task_slug":"code-translation","task_name":"Code Translation"},{"task_slug":"decoder","task_name":"Decoder"},{"task_slug":"defect-detection","task_name":"Defect Detection"},{"task_slug":"document-translation","task_name":"Document Translation"},{"task_slug":"text-to-code-generation","task_name":"Text-to-Code Generation"}],"methods":[],"datasets_introduced":[{"slug":"codexglue","name":"CodeXGLUE","full_name":""}],"methods_introduced":[],"results":[{"leaderboard":"/sota/cloze-test-on-codexglue-ct-all","task":"Cloze Test","dataset":"CodeXGLUE - CT-all","model":"CodeBERT(MLM)","rank_in_archive_order":1,"of":1,"metrics":{"Go":"83.31","JS":"81.77","Java":"80.63","PHP":"85.05","Python":"87.21","Ruby":"80.17"},"uses_additional_data":false},{"leaderboard":"/sota/cloze-test-on-codexglue-ct-maxmin","task":"Cloze Test","dataset":"CodeXGLUE - CT-maxmin","model":"CodeBERT(MLM)","rank_in_archive_order":1,"of":1,"metrics":{"Go":"90.79","JS":"86.4","Java":"90.46","PHP":"88.21","Python":"82.2","Ruby":"86.84"},"uses_additional_data":false},{"leaderboard":"/sota/code-completion-on-codexglue-github-java","task":"Code Completion","dataset":"CodeXGLUE - Github Java Corpus","model":"CodeGPT-adapted","rank_in_archive_order":1,"of":3,"metrics":{"Accuracy (token-level)":"77.13","EM (line-level)":"26.43","Edit Sim (line-level)":"63.03"},"uses_additional_data":false},{"leaderboard":"/sota/code-completion-on-codexglue-py150","task":"Code Completion","dataset":"CodeXGLUE - PY150","model":"CodeGPT-adapted","rank_in_archive_order":1,"of":3,"metrics":{"Accuracy (token-level)":"75.11","EM (line-level)":"39.65","Edit Sim (line-level)":"69.84"},"uses_additional_data":false},{"leaderboard":"/sota/code-repair-on-codexglue-bugs2fix","task":"Code Repair","dataset":"CodeXGLUE - Bugs2Fix","model":"CodeBERT","rank_in_archive_order":2,"of":2,"metrics":{"Accuracy (medium)":"5.2","Accuracy (small)":"16.4","BLEU (medium)":"91.07","BLEU (small)":"77.42","CodeBLEU (medium)":"87.52","CodeBLEU (small)":"75.58"},"uses_additional_data":false},{"leaderboard":"/sota/code-search-on-codexglue-advtest","task":"Code Search","dataset":"CodeXGLUE - AdvTest","model":"CodeBERT","rank_in_archive_order":3,"of":3,"metrics":{"MRR":"27.19"},"uses_additional_data":false},{"leaderboard":"/sota/code-search-on-codexglue-webquerytest","task":"Code Search","dataset":"CodeXGLUE - WebQueryTest","model":"CodeBERT","rank_in_archive_order":1,"of":1,"metrics":{"Accuracy":"47.8","F1":"58.95"},"uses_additional_data":false},{"leaderboard":"/sota/code-translation-on-codexglue-codetrans","task":"Code Translation","dataset":"CodeXGLUE - CodeTrans","model":"CodeBERT","rank_in_archive_order":2,"of":2,"metrics":{"Accuracy (C#→Java)":"58","Accuracy (Java→C#)":"59","BLEU (C#→Java)":"72.14","BLEU (Java→C#)":"79.92","CodeBLEU (C#→Java)":"79.41","CodeBLEU (Java→C#)":"85.1"},"uses_additional_data":false},{"leaderboard":"/sota/text-to-code-generation-on-codexglue-concode","task":"Text-to-Code Generation","dataset":"CodeXGLUE - CONCODE","model":"CodeGPT-adapted","rank_in_archive_order":2,"of":2,"metrics":{"BLEU":"32.79","CodeBLEU":"27.74","EM":"20.1"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2102.04664","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2102.04664"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Avmb/semantic_neq_game","reach":{"status":"unanswered"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kilimanj4r0/code-summarization-beyond-function-level","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yueyuel/programgen-lms-reliability","reach":{"status":"unanswered"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/CodeGen","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/deeplearnxmu/unigencoder","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/microsoft/CodeXGLUE","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sberbank-ai/fusion_brain_aij2021","reach":{"status":"unanswered"}}],"summary":{"ran_draft_wrong":1},"by_repo_kind":{"official":{"samples":1,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"5bd8d14c82f90c47","entry":"get_special_tokens","repo":"microsoft/CodeXGLUE","repo_kind":"official","path":"Code-Code/CodeCompletion-line/code/run_lm.py","file_url":"https://github.com/microsoft/CodeXGLUE/blob/HEAD/Code-Code/CodeCompletion-line/code/run_lm.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5bd8d14c82f90c47"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}