{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/catfood-counterfactual-augmented-training-for","title":"CATfOOD: Counterfactual Augmented Training for Improving Out-of-Domain Performance and Calibration","arxiv_id":"2309.07822","date":"2023-09-14","proceeding":null,"authors":["Rachneet Sachdeva","Martin Tutek","Iryna Gurevych"],"abstract":"In recent years, large language models (LLMs) have shown remarkable capabilities at scale, particularly at generating text conditioned on a prompt. In our work, we investigate the use of LLMs to augment training data of small language models~(SLMs) with automatically generated counterfactual~(CF) instances -- i.e. minimally altered inputs -- in order to improve out-of-domain~(OOD) performance of SLMs in the extractive question answering~(QA) setup. We show that, across various LLM generators, such data augmentation consistently enhances OOD performance and improves model calibration for both confidence-based and rationale-augmented calibrator models. Furthermore, these performance improvements correlate with higher diversity of CF instances in terms of their surface form and semantic content. Finally, we show that CF augmented models which are easier to calibrate also exhibit much lower entropy when assigning importance, indicating that rationale-augmented calibrators prefer concise explanations.","url_abs":"https://arxiv.org/abs/2309.07822v3","url_pdf":"https://arxiv.org/pdf/2309.07822v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"catfood-counterfactual-augmented-training-for","repo_url":"https://github.com/ukplab/catfood","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"data-augmentation","task_name":"Data Augmentation"},{"task_slug":"diversity","task_name":"Diversity"},{"task_slug":"extractive-question-answering","task_name":"Extractive Question-Answering"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":null,"task_name":"counterfactual"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2309.07822","atlas_url":"https://app.syntology.ai/?focus=2309.07822","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2309.07822"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ukplab/catfood","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":12},"by_repo_kind":{"official":{"samples":12,"ran":12,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"048fad94afc1d3f6","entry":"comprehensiveness","repo":"ukplab/catfood","repo_kind":"official","path":"src/faithfulness/faithfulness.py","file_url":"https://github.com/ukplab/catfood/blob/HEAD/src/faithfulness/faithfulness.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"048fad94afc1d3f6"}},{"code_sha256_prefix":"e35cb1727750ade1","entry":"exact_match_score","repo":"ukplab/catfood","repo_kind":"official","path":"src/shortcuts/evaluate.py","file_url":"https://github.com/ukplab/catfood/blob/HEAD/src/shortcuts/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e35cb1727750ade1"}},{"code_sha256_prefix":"ee7ee45b82d4949f","entry":"exact_match_score","repo":"ukplab/catfood","repo_kind":"official","path":"src/calibration/baseline/calibration_metrics.py","file_url":"https://github.com/ukplab/catfood/blob/HEAD/src/calibration/baseline/calibration_metrics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ee7ee45b82d4949f"}},{"code_sha256_prefix":"5e88f388713168c7","entry":"f1_score","repo":"ukplab/catfood","repo_kind":"official","path":"src/shortcuts/evaluate.py","file_url":"https://github.com/ukplab/catfood/blob/HEAD/src/shortcuts/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5e88f388713168c7"}},{"code_sha256_prefix":"8cf315af1221f8a1","entry":"f1_score","repo":"ukplab/catfood","repo_kind":"official","path":"src/calibration/baseline/calibration_metrics.py","file_url":"https://github.com/ukplab/catfood/blob/HEAD/src/calibration/baseline/calibration_metrics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8cf315af1221f8a1"}},{"code_sha256_prefix":"69a30240b39852df","entry":"get_dev_examples","repo":"ukplab/catfood","repo_kind":"official","path":"src/calibration/baseline/dataloader.py","file_url":"https://github.com/ukplab/catfood/blob/HEAD/src/calibration/baseline/dataloader.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"69a30240b39852df"}},{"code_sha256_prefix":"c89cf751ddcfdb07","entry":"load_bin","repo":"ukplab/catfood","repo_kind":"official","path":"src/shortcuts/eval_metrics.py","file_url":"https://github.com/ukplab/catfood/blob/HEAD/src/shortcuts/eval_metrics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c89cf751ddcfdb07"}},{"code_sha256_prefix":"20043e8d5730af16","entry":"load_torch_data","repo":"ukplab/catfood","repo_kind":"official","path":"src/shortcuts/eval_metrics.py","file_url":"https://github.com/ukplab/catfood/blob/HEAD/src/shortcuts/eval_metrics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"20043e8d5730af16"}},{"code_sha256_prefix":"89c5a532983ce170","entry":"normalize_answer","repo":"ukplab/catfood","repo_kind":"official","path":"src/shortcuts/evaluate.py","file_url":"https://github.com/ukplab/catfood/blob/HEAD/src/shortcuts/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"89c5a532983ce170"}},{"code_sha256_prefix":"4f656af1dfa55eb8","entry":"normalize_answer","repo":"ukplab/catfood","repo_kind":"official","path":"src/calibration/baseline/calibration_metrics.py","file_url":"https://github.com/ukplab/catfood/blob/HEAD/src/calibration/baseline/calibration_metrics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4f656af1dfa55eb8"}},{"code_sha256_prefix":"20e359e157b001eb","entry":"sufficiency","repo":"ukplab/catfood","repo_kind":"official","path":"src/faithfulness/faithfulness.py","file_url":"https://github.com/ukplab/catfood/blob/HEAD/src/faithfulness/faithfulness.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"20e359e157b001eb"}},{"code_sha256_prefix":"7eabcc356bd276af","entry":"top_k","repo":"ukplab/catfood","repo_kind":"official","path":"src/shortcuts/eval_metrics.py","file_url":"https://github.com/ukplab/catfood/blob/HEAD/src/shortcuts/eval_metrics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7eabcc356bd276af"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}