{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/discern-decoding-systematic-errors-in-natural","title":"DISCERN: Decoding Systematic Errors in Natural Language for Text Classifiers","arxiv_id":"2410.22239","date":"2024-10-29","proceeding":null,"authors":["Rakesh R. Menon","Shashank Srivastava"],"abstract":"Despite their high predictive accuracies, current machine learning systems often exhibit systematic biases stemming from annotation artifacts or insufficient support for certain classes in the dataset. Recent work proposes automatic methods for identifying and explaining systematic biases using keywords. We introduce DISCERN, a framework for interpreting systematic biases in text classifiers using language explanations. DISCERN iteratively generates precise natural language descriptions of systematic errors by employing an interactive loop between two large language models. Finally, we use the descriptions to improve classifiers by augmenting classifier training sets with synthetically generated instances or annotated examples via active learning. On three text-classification datasets, we demonstrate that language explanations from our framework induce consistent performance improvements that go beyond what is achievable with exemplars of systematic bias. Finally, in human evaluations, we show that users can interpret systematic biases more effectively (by over 25% relative) and efficiently when described through language explanations as opposed to cluster exemplars.","url_abs":"https://arxiv.org/abs/2410.22239v1","url_pdf":"https://arxiv.org/pdf/2410.22239v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"discern-decoding-systematic-errors-in-natural","repo_url":"https://github.com/rrmenon10/DISCERN","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":null}],"tasks":[{"task_slug":"active-learning","task_name":"Active Learning"},{"task_slug":"text-classification","task_name":"Text Classification"},{"task_slug":"text-classification-1","task_name":"text-classification"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2410.22239","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.22239"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/rrmenon10/DISCERN","reach":null}],"summary":{"ran":5,"ran_draft_wrong":1,"unverified":5},"by_repo_kind":{"official":{"samples":11,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":11,"samples":[{"code_sha256_prefix":"587e2ba5154f85b0","entry":"compute_score","repo":"rrmenon10/DISCERN","repo_kind":"official","path":"src/discern/refine.py","file_url":"https://github.com/rrmenon10/DISCERN/blob/HEAD/src/discern/refine.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"587e2ba5154f85b0"}},{"code_sha256_prefix":"de5671532f8618d7","entry":"get_description_score","repo":"rrmenon10/DISCERN","repo_kind":"official","path":"src/discern/refine.py","file_url":"https://github.com/rrmenon10/DISCERN/blob/HEAD/src/discern/refine.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"de5671532f8618d7"}},{"code_sha256_prefix":"478e1775f2ce81ae","entry":"get_prompt","repo":"rrmenon10/DISCERN","repo_kind":"official","path":"src/discern/refine.py","file_url":"https://github.com/rrmenon10/DISCERN/blob/HEAD/src/discern/refine.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"478e1775f2ce81ae"}},{"code_sha256_prefix":"e36c803f77711244","entry":"get_refine_prompt","repo":"rrmenon10/DISCERN","repo_kind":"official","path":"src/discern/refine.py","file_url":"https://github.com/rrmenon10/DISCERN/blob/HEAD/src/discern/refine.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e36c803f77711244"}},{"code_sha256_prefix":"e3d84502048184f2","entry":"make_exp_dir","repo":"rrmenon10/DISCERN","repo_kind":"official","path":"src/discern/refine.py","file_url":"https://github.com/rrmenon10/DISCERN/blob/HEAD/src/discern/refine.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e3d84502048184f2"}},{"code_sha256_prefix":"01a3c356acb12f97","entry":"setup_logging","repo":"rrmenon10/DISCERN","repo_kind":"official","path":"src/discern/refine.py","file_url":"https://github.com/rrmenon10/DISCERN/blob/HEAD/src/discern/refine.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"01a3c356acb12f97"}},{"code_sha256_prefix":"3f2d88d898f0a593","entry":"Cluster","repo":"rrmenon10/DISCERN","repo_kind":"official","path":"src/discern/refine.py","file_url":"https://github.com/rrmenon10/DISCERN/blob/HEAD/src/discern/refine.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3f2d88d898f0a593"}},{"code_sha256_prefix":"b9ba287d5dc3c1dd","entry":"Config","repo":"rrmenon10/DISCERN","repo_kind":"official","path":"src/discern/refine.py","file_url":"https://github.com/rrmenon10/DISCERN/blob/HEAD/src/discern/refine.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b9ba287d5dc3c1dd"}},{"code_sha256_prefix":"785ee891a47f53e1","entry":"first_description","repo":"rrmenon10/DISCERN","repo_kind":"official","path":"src/discern/refine.py","file_url":"https://github.com/rrmenon10/DISCERN/blob/HEAD/src/discern/refine.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"785ee891a47f53e1"}},{"code_sha256_prefix":"a3bc5649caed7187","entry":"refine_description","repo":"rrmenon10/DISCERN","repo_kind":"official","path":"src/discern/refine.py","file_url":"https://github.com/rrmenon10/DISCERN/blob/HEAD/src/discern/refine.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a3bc5649caed7187"}},{"code_sha256_prefix":"d0d47d603094440f","entry":"refinement","repo":"rrmenon10/DISCERN","repo_kind":"official","path":"src/discern/refine.py","file_url":"https://github.com/rrmenon10/DISCERN/blob/HEAD/src/discern/refine.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d0d47d603094440f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}