{"url":"/dataset/turbulence","name":"Turbulence","full_name":"Turbulence","description_markdown":"$\\textbf{Turbulence}$ is a new benchmark for systematically evaluating the correctness and robustness of instruction-tuned large language models (LLMs) for code generation. Turbulence consists of a large set of natural language question templates, each of which is a programming problem, parameterised so that it can be asked in many different forms. Each question template has an associated test oracle that judges whether a code solution returned by an LLM is correct. Thus, from a single question template, it is possible to ask an LLM a $\\textit{neighbourhood}$ of very similar programming questions, and assess the correctness of the result returned for each question. This new benchmark systematically and automatically identifies cases where LLMs are able to solve some problems in a neighbourhood but do not manage to generalise to solve the whole neighbourhood. Therefore, this method is effective at highlighting robustness issues.","description_withheld":null,"homepage":"https://github.com/ShahinHonarvar/Turbulence-Benchmark","introduced_date":"2023-12-22","introduced_date_note":null,"introduced_by":{"paper":"/paper/turbulence-systematically-and-automatically","title":"Turbulence: Systematically and Automatically Testing Instruction-Tuned Large Language Models for Code","first_author":"Shahin Honarvar","url":null},"license":{"name":"MIT","url":"https://github.com/ShahinHonarvar/Turbulence-Benchmark?tab=MIT-1-ov-file"},"modalities":[],"tasks":[{"name":"Code Generation","url":"/task/code-generation","datasets_with_task":"/datasets/task/code-generation"}],"languages":[],"variants":["Turbulence"],"data_loaders":[],"num_papers_in_archive":1,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28"},"benchmarks":[{"leaderboard":"/sota/code-generation-on-turbulence","task":"Code Generation","dataset_variant":"Turbulence","rows":5,"metrics":["CorrSc"],"first_row_in_archive_order":{"model":"GPT-4","paper":"/paper/turbulence-systematically-and-automatically","metrics":{"CorrSc":"0.848"},"code_links":[{"title":"shahinhonarvar/turbulence-benchmark","url":"https://github.com/shahinhonarvar/turbulence-benchmark"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"}],"papers_with_a_benchmark_row":[{"paper":"/paper/turbulence-systematically-and-automatically","title":"Turbulence: Systematically and Automatically Testing Instruction-Tuned Large Language Models for Code","date":"2023-12-22","rows_on_this_dataset":5,"code_links":1,"syntology":{"read_at":"2026-09-25T09:33:49+00:00","samples_harvested":4,"samples_ran":4,"samples_unverified":0,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}}],"syntology_totals":{"read_at":"2026-09-25T09:33:49+00:00","papers_with_samples":1,"samples_harvested":4,"samples_ran":4,"samples_unverified":0,"pointer_only_for_licence":0,"papers_with_no_sample_that_ran":0,"note":"the per-paper counts above, summed; not a rate"},"papers_note":"The archive never published its papers-using-dataset list; these are papers with a leaderboard row on this dataset's benchmarks."}