{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/semantic-evaluation-for-text-to-sql-with","title":"Semantic Evaluation for Text-to-SQL with Distilled Test Suites","arxiv_id":"2010.02840","date":"2020-10-06","proceeding":"EMNLP 2020 11","authors":["Ruiqi Zhong","Tao Yu","Dan Klein"],"abstract":"We propose test suite accuracy to approximate semantic accuracy for Text-to-SQL models. Our method distills a small test suite of databases that achieves high code coverage for the gold query from a large number of randomly generated databases. At evaluation time, it computes the denotation accuracy of the predicted queries on the distilled test suite, hence calculating a tight upper-bound for semantic accuracy efficiently. We use our proposed method to evaluate 21 models submitted to the Spider leader board and manually verify that our method is always correct on 100 examples. In contrast, the current Spider metric leads to a 2.5% false negative rate on average and 8.1% in the worst case, indicating that test suite accuracy is needed. Our implementation, along with distilled test suites for eleven Text-to-SQL datasets, is publicly available.","url_abs":"https://arxiv.org/abs/2010.02840v1","url_pdf":"https://arxiv.org/pdf/2010.02840v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"semantic-evaluation-for-text-to-sql-with","repo_url":"https://github.com/ruiqi-zhong/TestSuiteEval","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"semantic-evaluation-for-text-to-sql-with","repo_url":"https://github.com/taoyds/spider","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":null},{"paper_slug":"semantic-evaluation-for-text-to-sql-with","repo_url":"https://github.com/taoyds/test-suite-sql-eval","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":null}],"tasks":[{"task_slug":"text-to-sql","task_name":"Text to SQL"},{"task_slug":"text-to-sql","task_name":"Text-To-SQL"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2010.02840","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.02840"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ruiqi-zhong/TestSuiteEval","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/taoyds/spider","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/taoyds/test-suite-sql-eval","reach":null}],"summary":{"ran_fixture":1,"ran_violates":3,"ran_draft_wrong":1},"by_repo_kind":{"listed":{"samples":2,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":3,"samples":[{"code_sha256_prefix":"bd001f76894a9867","entry":"acc","repo":"taoyds/test-suite-sql-eval","repo_kind":"listed","path":"evaluate_classical.py","file_url":"https://github.com/taoyds/test-suite-sql-eval/blob/HEAD/evaluate_classical.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bd001f76894a9867"}},{"code_sha256_prefix":"8231e3fdffa9c8ee","entry":"condition_has_like","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"8231e3fdffa9c8ee"}},{"code_sha256_prefix":"f2d76234528d3b57","entry":"condition_has_or","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"f2d76234528d3b57"}},{"code_sha256_prefix":"954c76e256cc7220","entry":"condition_has_sql","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"954c76e256cc7220"}},{"code_sha256_prefix":"c64a2627f38e10a8","entry":"load_predictions","repo":"taoyds/test-suite-sql-eval","repo_kind":"listed","path":"evaluate_classical.py","file_url":"https://github.com/taoyds/test-suite-sql-eval/blob/HEAD/evaluate_classical.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c64a2627f38e10a8"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}