{"url":"/dataset/res-q","name":"RES-Q","full_name":"RES-Q: Evaluating Code-Editing Large Language Model Systems at the Repository Scale","description_markdown":"RES-Q is a natural language instruction-based benchmark for evaluating $\\textbf{R}$epository $\\textbf{E}$diting $\\textbf{S}$ystems, which consists of 100 handcrafted repository editing tasks derived from real GitHub commits. Given an edit instruction and a code repository, RES-Q evaluates an LLM system’s ability to interpret edit instructions, gather information, and construct appropriate edits to the repository.","description_withheld":null,"homepage":"https://github.com/Qurrent-AI/RES-Q","introduced_date":"2024-06-24","introduced_date_note":null,"introduced_by":{"paper":"/paper/res-q-evaluating-code-editing-large-language","title":"RES-Q: Evaluating Code-Editing Large Language Model Systems at the Repository Scale","first_author":"Beck Labash","url":null},"license":{"name":"MIT","url":"https://github.com/Qurrent-AI/RES-Q/blob/main/LICENSE.md"},"modalities":[{"name":"Texts","url":"/datasets/modality/texts"}],"tasks":[{"name":"Code Generation","url":"/task/code-generation","datasets_with_task":"/datasets/task/code-generation"},{"name":"Code Search","url":"/task/code-search","datasets_with_task":"/datasets/task/code-search"},{"name":"Text-to-Code Generation","url":"/task/text-to-code-generation","datasets_with_task":"/datasets/task/text-to-code-generation"},{"name":"Library-Oriented Code Generation","url":"/task/library-oriented-code-generation","datasets_with_task":"/datasets/task/library-oriented-code-generation"}],"languages":[{"name":"English","url":"/datasets/language/english"}],"variants":["RES-Q"],"data_loaders":[],"num_papers_in_archive":1,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28"},"benchmarks":[{"leaderboard":"/sota/code-generation-on-res-q","task":"Code Generation","dataset_variant":"RES-Q","rows":9,"metrics":["pass@1"],"first_row_in_archive_order":{"model":"QurrentOS-coder + Claude 3.5 Sonnet","paper":"/paper/res-q-evaluating-code-editing-large-language","metrics":{"pass@1":"58.0"},"code_links":[{"title":"qurrent-ai/res-q","url":"https://github.com/qurrent-ai/res-q"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"}],"papers_with_a_benchmark_row":[{"paper":"/paper/res-q-evaluating-code-editing-large-language","title":"RES-Q: Evaluating Code-Editing Large Language Model Systems at the Repository Scale","date":"2024-06-24","rows_on_this_dataset":9,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":6,"samples_ran":6,"samples_unverified":0,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}}],"syntology_totals":{"read_at":"2026-09-24T18:15:14+00:00","papers_with_samples":1,"samples_harvested":6,"samples_ran":6,"samples_unverified":0,"pointer_only_for_licence":0,"papers_with_no_sample_that_ran":0,"note":"the per-paper counts above, summed; not a rate"},"papers_note":"The archive never published its papers-using-dataset list; these are papers with a leaderboard row on this dataset's benchmarks."}