{"url":"/dataset/conala","name":"CoNaLa","full_name":"CMU CoNaLa, the Code/Natural Language Challenge","description_markdown":"The **CMU CoNaLa, the Code/Natural Language Challenge** dataset is a joint project from the Carnegie Mellon University [NeuLab](http://www.cs.cmu.edu/~neulab/) and [Strudel](https://cmustrudel.github.io/) labs. Its purpose is for testing the generation of code snippets from natural language. The data comes from StackOverflow questions. There are 2379 training and 500 test examples that were manually annotated. Every example has a natural language *intent* and its corresponding python *snippet*.  In addition to the manually annotated dataset, there are also 598,237 mined intent-snippet pairs. These examples are similar to the hand-annotated ones except that they contain a probability if the pair is valid.\r\n\r\nSource: [CoNaLa dataset Homepage](https://conala-corpus.github.io/)","description_withheld":null,"homepage":"https://conala-corpus.github.io/","introduced_date":"2018-05-23","introduced_date_note":null,"introduced_by":{"paper":"/paper/learning-to-mine-aligned-code-and-natural","title":"Learning to Mine Aligned Code and Natural Language Pairs from Stack Overflow","first_author":"Pengcheng Yin","url":null},"license":null,"modalities":[{"name":"Texts","url":"/datasets/modality/texts"}],"tasks":[{"name":"Code Generation","url":"/task/code-generation","datasets_with_task":"/datasets/task/code-generation"},{"name":"Code Search","url":"/task/code-search","datasets_with_task":"/datasets/task/code-search"}],"languages":[{"name":"English","url":"/datasets/language/english"},{"name":"Chinese","url":"/datasets/language/chinese"}],"variants":["CoNaLa"],"data_loaders":[],"num_papers_in_archive":77,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28"},"benchmarks":[{"leaderboard":"/sota/code-generation-on-conala","task":"Code Generation","dataset_variant":"CoNaLa","rows":14,"metrics":["BLEU","Exact Match Accuracy"],"first_row_in_archive_order":{"model":"PanGu-Coder-FT-I","paper":"/paper/fine-tuning-large-language-models-for","metrics":{"BLEU":"44.32"},"code_links":[]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"}],"papers_with_a_benchmark_row":[{"paper":"/paper/leveraging-pre-trained-language-models-for-3","title":"Leveraging pre-trained language models for code generation","date":"2024-02-29","rows_on_this_dataset":4,"code_links":1,"syntology":null},{"paper":"/paper/fine-tuning-large-language-models-for","title":"Fine-Tuning Large Language Models for Answering Programming Questions with Code Snippets","date":"2023-06-26","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/mariancg-a-code-generation-transformer-model","title":"MarianCG: a code generation transformer model inspired by machine translation","date":"2022-11-22","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/the-impact-of-lexical-and-grammatical-1","title":"The impact of lexical and grammatical processing on generating code from natural language","date":"2022-02-28","rows_on_this_dataset":1,"code_links":2,"syntology":null},{"paper":"/paper/reading-stackoverflow-encourages-cheating","title":"Reading StackOverflow Encourages Cheating: Adding Question Text Improves Extractive Code Generation","date":"2021-06-08","rows_on_this_dataset":2,"code_links":1,"syntology":null},{"paper":"/paper/semantic-parsing-with-less-prior-and-more","title":"Code Generation from Natural Language with Less Prior and More Monolingual Data","date":"2021-01-01","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/incorporating-external-knowledge-through-pre","title":"Incorporating External Knowledge through Pre-training for Natural Language to Code Generation","date":"2020-04-20","rows_on_this_dataset":2,"code_links":2,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":5,"samples_ran":0,"samples_unverified":5,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/reranking-for-neural-semantic-parsing","title":"Reranking for Neural Semantic Parsing","date":"2019-07-01","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/tranx-a-transition-based-neural-abstract","title":"TRANX: A Transition-based Neural Abstract Syntax Parser for Semantic Parsing and Code Generation","date":"2018-10-05","rows_on_this_dataset":1,"code_links":4,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":7,"samples_ran":2,"samples_unverified":5,"pointer_only_for_licence":2,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}}],"syntology_totals":{"read_at":"2026-09-24T18:15:14+00:00","papers_with_samples":2,"samples_harvested":12,"samples_ran":2,"samples_unverified":10,"pointer_only_for_licence":2,"papers_with_no_sample_that_ran":1,"note":"the per-paper counts above, summed; not a rate"},"papers_note":"The archive never published its papers-using-dataset list; these are papers with a leaderboard row on this dataset's benchmarks."}