{"url":"/dataset/spider-realistic","name":"Spider-Realistic","full_name":null,"description_markdown":"Spider dataset is used for evaluation in the paper \"Structure-Grounded Pretraining for Text-to-SQL\". The dataset is created based on the dev split of the Spider dataset (2020-06-07 version from https://yale-lily.github.io/spider). We manually modified the original questions to remove the explicit mention of column names while keeping the SQL queries unchanged to better evaluate the model's capability in aligning the NL utterance and the DB schema. For more details, please check our paper at https://arxiv.org/abs/2010.12773.","description_withheld":null,"homepage":"https://zenodo.org/record/5205322#.YTts_o5Kgab","introduced_date":"2020-10-24","introduced_date_note":null,"introduced_by":{"paper":"/paper/structure-grounded-pretraining-for-text-to","title":"Structure-Grounded Pretraining for Text-to-SQL","first_author":"Xiang Deng","url":null},"license":null,"modalities":[],"tasks":[{"name":"Semantic Parsing","url":"/task/semantic-parsing","datasets_with_task":"/datasets/task/semantic-parsing"},{"name":"Text-To-SQL","url":"/task/text-to-sql","datasets_with_task":"/datasets/task/text-to-sql"}],"languages":[],"variants":["spider","Spider-Realistic"],"data_loaders":[],"num_papers_in_archive":91,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28"},"benchmarks":[{"leaderboard":"/sota/text-to-sql-on-spider","task":"Text-To-SQL","dataset_variant":"spider","rows":20,"metrics":["Execution Accuracy (Test)","Exact Match Accuracy (Test)","Execution Accuracy (Dev)","Exact Match Accuracy (Dev)"],"first_row_in_archive_order":{"model":"XiYan-SQL","paper":"/paper/xiyan-sql-a-multi-generator-ensemble","metrics":{"Execution Accuracy (Test)":"89.65"},"code_links":[{"title":"XGenerationLab/XiYan-SQL","url":"https://github.com/XGenerationLab/XiYan-SQL"},{"title":"xgenerationlab/xiyan_mcp_server","url":"https://github.com/xgenerationlab/xiyan_mcp_server"},{"title":"XGenerationLab/M-Schema","url":"https://github.com/XGenerationLab/M-Schema"},{"title":"xgenerationlab/xiyan-dbdescgen","url":"https://github.com/xgenerationlab/xiyan-dbdescgen"},{"title":"XGenerationLab/XiYan-DateResolver","url":"https://github.com/XGenerationLab/XiYan-DateResolver"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/semantic-parsing-on-spider","task":"Semantic Parsing","dataset_variant":"spider","rows":10,"metrics":["Accuracy"],"first_row_in_archive_order":{"model":"RESDSQL-3B + NatSQL","paper":"/paper/decoupling-the-skeleton-parsing-and-schema","metrics":{"Accuracy":"84.1"},"code_links":[{"title":"ruckbreasoning/resdsql","url":"https://github.com/ruckbreasoning/resdsql"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/text-to-sql-on-spider-1","task":"Text-To-SQL","dataset_variant":"SPIDER","rows":4,"metrics":["Exact Match Accuracy (in Dev)","Execution Accuracy (in Dev)"],"first_row_in_archive_order":{"model":"T5-3B+PICARD","paper":"/paper/picard-parsing-incrementally-for-constrained","metrics":{"Exact Match Accuracy (in Dev)":"75.5","Execution Accuracy (in Dev)":"79.3"},"code_links":[{"title":"servicenow/picard","url":"https://github.com/servicenow/picard"},{"title":"ElementAI/picard","url":"https://github.com/ElementAI/picard"},{"title":"yangyucheng000/Paper-3","url":"https://github.com/yangyucheng000/Paper-3/tree/main/acm-mm-2023-picr-master"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"}],"papers_with_a_benchmark_row":[{"paper":"/paper/xiyan-sql-a-multi-generator-ensemble","title":"A Preview of XiYan-SQL: A Multi-Generator Ensemble Framework for Text-to-SQL","date":"2024-11-13","rows_on_this_dataset":1,"code_links":5,"syntology":null},{"paper":"/paper/learning-metadata-agnostic-representations","title":"Learning Metadata-Agnostic Representations for Text-to-SQL In-Context Example Selection","date":"2024-10-17","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/msc-sql-multi-sample-critiquing-small","title":"MSc-SQL: Multi-Sample Critiquing Small Language Models For Text-To-SQL Translation","date":"2024-10-16","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":9,"samples_ran":9,"samples_unverified":0,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/datagpt-sql-7b-an-open-source-language-model-1","title":"DataGpt-SQL-7B: An Open-Source Language Model for Text-to-SQL","date":"2024-09-24","rows_on_this_dataset":2,"code_links":0,"syntology":null},{"paper":"/paper/pet-sql-a-prompt-enhanced-two-stage-text-to","title":"PET-SQL: A Prompt-Enhanced Two-Round Refinement of Text-to-SQL with Cross-consistency","date":"2024-03-13","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":9,"samples_ran":6,"samples_unverified":3,"pointer_only_for_licence":9,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/knowledge-to-sql-enhancing-sql-generation","title":"Knowledge-to-SQL: Enhancing SQL Generation with Data Expert LLM","date":"2024-02-18","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":3,"samples_ran":2,"samples_unverified":1,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/text-to-sql-empowered-by-large-language","title":"Text-to-SQL Empowered by Large Language Models: A Benchmark Evaluation","date":"2023-08-29","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":3,"samples_ran":1,"samples_unverified":2,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/c3-zero-shot-text-to-sql-with-chatgpt","title":"C3: Zero-shot Text-to-SQL with ChatGPT","date":"2023-07-14","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":2,"samples_ran":2,"samples_unverified":0,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/t5-sr-a-unified-seq-to-seq-decoding-strategy","title":"T5-SR: A Unified Seq-to-Seq Decoding Strategy for Semantic Parsing","date":"2023-06-14","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/improving-generalization-in-language-model","title":"Improving Generalization in Language Model-Based Text-to-SQL Semantic Parsing: Two Simple Semantic Boundary-Based Techniques","date":"2023-05-27","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/din-sql-decomposed-in-context-learning-of-1","title":"DIN-SQL: Decomposed In-Context Learning of Text-to-SQL with Self-Correction","date":"2023-04-21","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":4,"samples_ran":0,"samples_unverified":4,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/lever-learning-to-verify-language-to-code","title":"LEVER: Learning to Verify Language-to-Code Generation with Execution","date":"2023-02-16","rows_on_this_dataset":2,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":22,"samples_ran":6,"samples_unverified":16,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/decoupling-the-skeleton-parsing-and-schema","title":"RESDSQL: Decoupling Schema Linking and Skeleton Parsing for Text-to-SQL","date":"2023-02-12","rows_on_this_dataset":2,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":1,"samples_ran":0,"samples_unverified":1,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/graphix-t5-mixing-pre-trained-transformers","title":"Graphix-T5: Mixing Pre-Trained Transformers with Graph-Aware Layers for Text-to-SQL Parsing","date":"2023-01-18","rows_on_this_dataset":2,"code_links":1,"syntology":null},{"paper":"/paper/rasat-integrating-relational-structures-into","title":"RASAT: Integrating Relational Structures into Pretrained Seq2Seq Model for Text-to-SQL","date":"2022-05-14","rows_on_this_dataset":3,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":9,"samples_ran":0,"samples_unverified":9,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/sadga-structure-aware-dual-graph-aggregation","title":"SADGA: Structure-Aware Dual Graph Aggregation Network for Text-to-SQL","date":"2021-11-01","rows_on_this_dataset":2,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":7,"samples_ran":5,"samples_unverified":2,"pointer_only_for_licence":7,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/picard-parsing-incrementally-for-constrained","title":"PICARD: Parsing Incrementally for Constrained Auto-Regressive Decoding from Language Models","date":"2021-09-10","rows_on_this_dataset":4,"code_links":3,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":7,"samples_ran":4,"samples_unverified":3,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/learning-contextual-representations-for","title":"Learning Contextual Representations for Semantic Parsing with Generation-Augmented Pre-Training","date":"2020-12-18","rows_on_this_dataset":2,"code_links":3,"syntology":null},{"paper":"/paper/grappa-grammar-augmented-pre-training-for","title":"GraPPa: Grammar-Augmented Pre-Training for Table Semantic Parsing","date":"2020-09-29","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":7,"samples_ran":6,"samples_unverified":1,"pointer_only_for_licence":7,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/tabert-pretraining-for-joint-understanding-of","title":"TaBERT: Pretraining for Joint Understanding of Textual and Tabular Data","date":"2020-05-17","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/ryansql-recursively-applying-sketch-based","title":"RYANSQL: Recursively Applying Sketch-based Slot Fillings for Complex Text-to-SQL in Cross-Domain Databases","date":"2020-04-07","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":2,"samples_ran":1,"samples_unverified":1,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/rat-sql-relation-aware-schema-encoding-and-1","title":"RAT-SQL: Relation-Aware Schema Encoding and Linking for Text-to-SQL Parsers","date":"2019-11-10","rows_on_this_dataset":1,"code_links":4,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":7,"samples_ran":4,"samples_unverified":3,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/spider-a-large-scale-human-labeled-dataset","title":"Spider: A Large-Scale Human-Labeled Dataset for Complex and Cross-Domain Semantic Parsing and Text-to-SQL Task","date":"2018-09-24","rows_on_this_dataset":1,"code_links":6,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":8,"samples_ran":7,"samples_unverified":1,"pointer_only_for_licence":3,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}}],"syntology_totals":{"read_at":"2026-09-24T18:15:14+00:00","papers_with_samples":15,"samples_harvested":100,"samples_ran":53,"samples_unverified":47,"pointer_only_for_licence":26,"papers_with_no_sample_that_ran":3,"note":"the per-paper counts above, summed; not a rate"},"papers_note":"The archive never published its papers-using-dataset list; these are papers with a leaderboard row on this dataset's benchmarks."}