{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/towards-robustness-of-text-to-sql-models-3","title":"Towards Robustness of Text-to-SQL Models Against Natural and Realistic Adversarial Table Perturbation","arxiv_id":"2212.09994","date":"2022-12-20","proceeding":"ACL 2022 5","authors":["Xinyu Pi","Bing Wang","Yan Gao","Jiaqi Guo","Zhoujun Li","Jian-Guang Lou"],"abstract":"The robustness of Text-to-SQL parsers against adversarial perturbations plays a crucial role in delivering highly reliable applications. Previous studies along this line primarily focused on perturbations in the natural language question side, neglecting the variability of tables. Motivated by this, we propose the Adversarial Table Perturbation (ATP) as a new attacking paradigm to measure the robustness of Text-to-SQL models. Following this proposition, we curate ADVETA, the first robustness evaluation benchmark featuring natural and realistic ATPs. All tested state-of-the-art models experience dramatic performance drops on ADVETA, revealing models' vulnerability in real-world practices. To defend against ATP, we build a systematic adversarial training example generation framework tailored for better contextualization of tabular data. Experiments show that our approach not only brings the best robustness improvement against table-side perturbations but also substantially empowers models against NL-side perturbations. We release our benchmark and code at: https://github.com/microsoft/ContextualSP.","url_abs":"https://arxiv.org/abs/2212.09994v1","url_pdf":"https://arxiv.org/pdf/2212.09994v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"towards-robustness-of-text-to-sql-models-3","repo_url":"https://github.com/microsoft/ContextualSP","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"text-to-sql","task_name":"Text to SQL"},{"task_slug":"text-to-sql","task_name":"Text-To-SQL"}],"methods":[],"datasets_introduced":[{"slug":"adveta","name":"ADVETA","full_name":""}],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2212.09994","atlas_url":"https://app.syntology.ai/?focus=2212.09994","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2212.09994"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/microsoft/ContextualSP","reach":null}],"summary":{"ran":1,"ran_honours":1,"ran_fixture":1,"unverified":2},"by_repo_kind":{"official":{"samples":5,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"e0be17029a345e06","entry":"SmartPerturbation","repo":"microsoft/ContextualSP","repo_kind":"official","path":"adaptershare/mt_dnn/perturbation.py","file_url":"https://github.com/microsoft/ContextualSP/blob/HEAD/adaptershare/mt_dnn/perturbation.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e0be17029a345e06"}},{"code_sha256_prefix":"417ac95acb8c20b4","entry":"generate_noise","repo":"microsoft/ContextualSP","repo_kind":"official","path":"adaptershare/mt_dnn/perturbation.py","file_url":"https://github.com/microsoft/ContextualSP/blob/HEAD/adaptershare/mt_dnn/perturbation.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"417ac95acb8c20b4"}},{"code_sha256_prefix":"7b0111630cee9629","entry":"stable_kl","repo":"microsoft/ContextualSP","repo_kind":"official","path":"adaptershare/mt_dnn/perturbation.py","file_url":"https://github.com/microsoft/ContextualSP/blob/HEAD/adaptershare/mt_dnn/perturbation.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7b0111630cee9629"}},{"code_sha256_prefix":"16d7e293217bcfda","entry":"EncoderModelType","repo":"microsoft/ContextualSP","repo_kind":"official","path":"adaptershare/mt_dnn/perturbation.py","file_url":"https://github.com/microsoft/ContextualSP/blob/HEAD/adaptershare/mt_dnn/perturbation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"16d7e293217bcfda"}},{"code_sha256_prefix":"9c21f574753121d3","entry":"TaskType","repo":"microsoft/ContextualSP","repo_kind":"official","path":"adaptershare/mt_dnn/perturbation.py","file_url":"https://github.com/microsoft/ContextualSP/blob/HEAD/adaptershare/mt_dnn/perturbation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9c21f574753121d3"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}