{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/coco-counterfactuals-automatically","title":"COCO-Counterfactuals: Automatically Constructed Counterfactual Examples for Image-Text Pairs","arxiv_id":"2309.14356","date":"2023-09-23","proceeding":"NeurIPS 2023 11","authors":[],"abstract":"Counterfactual examples have proven to be valuable in the field of natural language processing (NLP) for both evaluating and improving the robustness of language models to spurious correlations in datasets. Despite their demonstrated utility for NLP, multimodal counterfactual examples have been relatively unexplored due to the difficulty of creating paired image-text data with minimal counterfactual changes. To address this challenge, we introduce a scalable framework for automatic generation of counterfactual examples using text-to-image diffusion models. We use our framework to create COCO-Counterfactuals, a multimodal counterfactual dataset of paired image and text captions based on the MS-COCO dataset. We validate the quality of COCO-Counterfactuals through human evaluations and show that existing multimodal models are challenged by our counterfactual image-text pairs. Additionally, we demonstrate the usefulness of COCO-Counterfactuals for improving out-of-domain generalization of multimodal vision-language models via training data augmentation.","url_abs":"https://arxiv.org/abs/2309.14356v2","url_pdf":"https://arxiv.org/pdf/2309.14356v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[],"tasks":[{"task_slug":"data-augmentation","task_name":"Data Augmentation"},{"task_slug":"domain-generalization","task_name":"Domain Generalization"},{"task_slug":null,"task_name":"counterfactual"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2309.14356","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2309.14356"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/huggingface/transformers","reach":null},{"provenance":"deterministic:regex_extraction","url":"https://github.com/UKPLab/sentence-transformers","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/timothybrooks/instruct-pix2pix","reach":{"status":"ok","spdx":"NOASSERTION"}}],"summary":{"ran_fixture":1,"ran":4,"unverified":3},"by_repo_kind":{"found_in_text":{"samples":8,"ran":5,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":5,"samples":[{"code_sha256_prefix":"30befb7e4327e615","entry":"append_dims","repo":"timothybrooks/instruct-pix2pix","repo_kind":"found_in_text","path":"dataset_creation/generate_img_dataset.py","file_url":"https://github.com/timothybrooks/instruct-pix2pix/blob/HEAD/dataset_creation/generate_img_dataset.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":2,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"30befb7e4327e615"}},{"code_sha256_prefix":"e12c9d90a7538793","entry":"format_duration","repo":"UKPLab/sentence-transformers","repo_kind":"found_in_text","path":"sentence_transformers/base/model_card.py","file_url":"https://github.com/UKPLab/sentence-transformers/blob/HEAD/sentence_transformers/base/model_card.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e12c9d90a7538793"}},{"code_sha256_prefix":"fd50038f6df25d57","entry":"format_log","repo":"UKPLab/sentence-transformers","repo_kind":"found_in_text","path":"sentence_transformers/base/model_card.py","file_url":"https://github.com/UKPLab/sentence-transformers/blob/HEAD/sentence_transformers/base/model_card.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fd50038f6df25d57"}},{"code_sha256_prefix":"1921ebc84491b22c","entry":"get_ancestral_step","repo":"timothybrooks/instruct-pix2pix","repo_kind":"found_in_text","path":"dataset_creation/generate_img_dataset.py","file_url":"https://github.com/timothybrooks/instruct-pix2pix/blob/HEAD/dataset_creation/generate_img_dataset.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"1921ebc84491b22c"}},{"code_sha256_prefix":"2a68339364737323","entry":"get_losses","repo":"UKPLab/sentence-transformers","repo_kind":"found_in_text","path":"sentence_transformers/base/model_card.py","file_url":"https://github.com/UKPLab/sentence-transformers/blob/HEAD/sentence_transformers/base/model_card.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2a68339364737323"}},{"code_sha256_prefix":"4533e43e06a12dc5","entry":"download_models","repo":"timothybrooks/instruct-pix2pix","repo_kind":"found_in_text","path":"stable_diffusion/notebook_helpers.py","file_url":"https://github.com/timothybrooks/instruct-pix2pix/blob/HEAD/stable_diffusion/notebook_helpers.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4533e43e06a12dc5"}},{"code_sha256_prefix":"096488ca08edf235","entry":"generate","repo":"timothybrooks/instruct-pix2pix","repo_kind":"found_in_text","path":"dataset_creation/generate_txt_dataset.py","file_url":"https://github.com/timothybrooks/instruct-pix2pix/blob/HEAD/dataset_creation/generate_txt_dataset.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"096488ca08edf235"}},{"code_sha256_prefix":"042cd250dc4b2a67","entry":"to_d","repo":"timothybrooks/instruct-pix2pix","repo_kind":"found_in_text","path":"dataset_creation/generate_img_dataset.py","file_url":"https://github.com/timothybrooks/instruct-pix2pix/blob/HEAD/dataset_creation/generate_img_dataset.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"042cd250dc4b2a67"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}