{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/the-dawn-of-natural-language-to-sql-are-we","title":"The Dawn of Natural Language to SQL: Are We Fully Ready?","arxiv_id":"2406.01265","date":"2024-06-03","proceeding":null,"authors":["Boyan Li","Yuyu Luo","Chengliang Chai","Guoliang Li","Nan Tang"],"abstract":"Translating users' natural language questions into SQL queries (i.e., NL2SQL) significantly lowers the barriers to accessing relational databases. The emergence of Large Language Models has introduced a novel paradigm in NL2SQL tasks, enhancing capabilities dramatically. However, this raises a critical question: Are we fully prepared to deploy NL2SQL models in production? To address the posed questions, we present a multi-angle NL2SQL evaluation framework, NL2SQL360, to facilitate the design and test of new NL2SQL methods for researchers. Through NL2SQL360, we conduct a detailed comparison of leading NL2SQL methods across a range of application scenarios, such as different data domains and SQL characteristics, offering valuable insights for selecting the most appropriate NL2SQL methods for specific needs. Moreover, we explore the NL2SQL design space, leveraging NL2SQL360 to automate the identification of an optimal NL2SQL solution tailored to user-specific needs. Specifically, NL2SQL360 identifies an effective NL2SQL method, SuperSQL, distinguished under the Spdier dataset using the execution accuracy metric. Remarkably, SuperSQL achieves competitive performance with execution accuracy of 87% and 62.66% on the Spider and BIRD test sets, respectively.","url_abs":"https://arxiv.org/abs/2406.01265v3","url_pdf":"https://arxiv.org/pdf/2406.01265v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"links_only","authors_date_abstract":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license), from the Kaggle arXiv metadata snapshot of 2026-09-12"},"code_links":[{"paper_slug":"the-dawn-of-natural-language-to-sql-are-we","repo_url":"https://github.com/hkustdial/nl2sql360","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"the-dawn-of-natural-language-to-sql-are-we","repo_url":"https://github.com/hkustdial/nl2sql_survey","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"the-dawn-of-natural-language-to-sql-are-we","repo_url":"https://github.com/BugMaker-Boyan/NL2SQL360","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2406.01265","atlas_url":"https://app.syntology.ai/?focus=2406.01265","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.01265"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/hkustdial/nl2sql_survey","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/BugMaker-Boyan/NL2SQL360","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/hkustdial/nl2sql360","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":8},"by_repo_kind":{"official":{"samples":8,"ran":8,"repositories":3}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":1,"samples":[{"code_sha256_prefix":"01f8212987be7de4","entry":"generate_report_database_complexity","repo":"hkustdial/nl2sql_survey","repo_kind":"official","path":"src/dataset_analyze/utils.py","file_url":"https://github.com/hkustdial/nl2sql_survey/blob/HEAD/src/dataset_analyze/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"01f8212987be7de4"}},{"code_sha256_prefix":"dfb2b1550b67fd18","entry":"get_dataset_model","repo":"BugMaker-Boyan/NL2SQL360","repo_kind":"official","path":"src/nl2sql360/database/model.py","file_url":"https://github.com/BugMaker-Boyan/NL2SQL360/blob/HEAD/src/nl2sql360/database/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dfb2b1550b67fd18"}},{"code_sha256_prefix":"422e666a24b8a5a2","entry":"get_dataset_name_and_evaluation_name_from_table_name","repo":"BugMaker-Boyan/NL2SQL360","repo_kind":"official","path":"src/nl2sql360/database/util.py","file_url":"https://github.com/BugMaker-Boyan/NL2SQL360/blob/HEAD/src/nl2sql360/database/util.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"422e666a24b8a5a2"}},{"code_sha256_prefix":"fe965a6f81819e38","entry":"get_dataset_name_from_table_name","repo":"BugMaker-Boyan/NL2SQL360","repo_kind":"official","path":"src/nl2sql360/database/util.py","file_url":"https://github.com/BugMaker-Boyan/NL2SQL360/blob/HEAD/src/nl2sql360/database/util.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fe965a6f81819e38"}},{"code_sha256_prefix":"00d2cec512f23e7f","entry":"get_evaluation_model","repo":"BugMaker-Boyan/NL2SQL360","repo_kind":"official","path":"src/nl2sql360/database/model.py","file_url":"https://github.com/BugMaker-Boyan/NL2SQL360/blob/HEAD/src/nl2sql360/database/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"00d2cec512f23e7f"}},{"code_sha256_prefix":"31d52a91b5601210","entry":"get_evaluation_model","repo":"hkustdial/nl2sql360","repo_kind":"official","path":"src/nl2sql360/database/model.py","file_url":"https://github.com/hkustdial/nl2sql360/blob/HEAD/src/nl2sql360/database/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"31d52a91b5601210"}},{"code_sha256_prefix":"95928b0df03a9fa7","entry":"make_choice_type_function","repo":"BugMaker-Boyan/NL2SQL360","repo_kind":"official","path":"src/nl2sql360/arguments/hf_argparser.py","file_url":"https://github.com/BugMaker-Boyan/NL2SQL360/blob/HEAD/src/nl2sql360/arguments/hf_argparser.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"95928b0df03a9fa7"}},{"code_sha256_prefix":"0f5e06900e868c0f","entry":"string_to_bool","repo":"BugMaker-Boyan/NL2SQL360","repo_kind":"official","path":"src/nl2sql360/arguments/hf_argparser.py","file_url":"https://github.com/BugMaker-Boyan/NL2SQL360/blob/HEAD/src/nl2sql360/arguments/hf_argparser.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0f5e06900e868c0f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}