{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/the-unmet-promise-of-synthetic-training","title":"The Unmet Promise of Synthetic Training Images: Using Retrieved Real Images Performs Better","arxiv_id":"2406.05184","date":"2024-06-07","proceeding":null,"authors":["Scott Geng","Cheng-Yu Hsieh","Vivek Ramanujan","Matthew Wallingford","Chun-Liang Li","Pang Wei Koh","Ranjay Krishna"],"abstract":"Generative text-to-image models enable us to synthesize unlimited amounts of images in a controllable manner, spurring many recent efforts to train vision models with synthetic data. However, every synthetic image ultimately originates from the upstream data used to train the generator. Does the intermediate generator provide additional information over directly training on relevant parts of the upstream data? Grounding this question in the setting of image classification, we compare finetuning on task-relevant, targeted synthetic data generated by Stable Diffusion -- a generative model trained on the LAION-2B dataset -- against finetuning on targeted real images retrieved directly from LAION-2B. We show that while synthetic data can benefit some downstream tasks, it is universally matched or outperformed by real data from the simple retrieval baseline. Our analysis suggests that this underperformance is partially due to generator artifacts and inaccurate task-relevant visual details in the synthetic images. Overall, we argue that targeted retrieval is a critical baseline to consider when training with synthetic data -- a baseline that current methods do not yet surpass. We release code, data, and models at https://github.com/scottgeng00/unmet-promise.","url_abs":"https://arxiv.org/abs/2406.05184v5","url_pdf":"https://arxiv.org/pdf/2406.05184v5.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"the-unmet-promise-of-synthetic-training","repo_url":"https://github.com/scottgeng00/unmet-promise","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"image-classification","task_name":"Image Classification"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"image-classification","task_name":"image-classification"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2406.05184","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.05184"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/scottgeng00/unmet-promise","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":4,"ran":1,"unverified":4},"by_repo_kind":{"official":{"samples":9,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"98f385d847636a3e","entry":"basic_clean","repo":"scottgeng00/unmet-promise","repo_kind":"official","path":"adapt/clip_custom/tokenizer.py","file_url":"https://github.com/scottgeng00/unmet-promise/blob/HEAD/adapt/clip_custom/tokenizer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"98f385d847636a3e"}},{"code_sha256_prefix":"6d77ee840479c381","entry":"convert","repo":"scottgeng00/unmet-promise","repo_kind":"official","path":"adapt/src/datasets/cifar10.py","file_url":"https://github.com/scottgeng00/unmet-promise/blob/HEAD/adapt/src/datasets/cifar10.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6d77ee840479c381"}},{"code_sha256_prefix":"d919ae32e5e4e616","entry":"get_pairs","repo":"scottgeng00/unmet-promise","repo_kind":"official","path":"adapt/clip_custom/tokenizer.py","file_url":"https://github.com/scottgeng00/unmet-promise/blob/HEAD/adapt/clip_custom/tokenizer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d919ae32e5e4e616"}},{"code_sha256_prefix":"b1e9e0d3d7d7e615","entry":"maybe_dictionarize","repo":"scottgeng00/unmet-promise","repo_kind":"official","path":"adapt/src/datasets/common.py","file_url":"https://github.com/scottgeng00/unmet-promise/blob/HEAD/adapt/src/datasets/common.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b1e9e0d3d7d7e615"}},{"code_sha256_prefix":"9542161e9640b858","entry":"whitespace_clean","repo":"scottgeng00/unmet-promise","repo_kind":"official","path":"adapt/clip_custom/tokenizer.py","file_url":"https://github.com/scottgeng00/unmet-promise/blob/HEAD/adapt/clip_custom/tokenizer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9542161e9640b858"}},{"code_sha256_prefix":"4e1f6a9710dc6279","entry":"build_model","repo":"scottgeng00/unmet-promise","repo_kind":"official","path":"adapt/clip_custom/model.py","file_url":"https://github.com/scottgeng00/unmet-promise/blob/HEAD/adapt/clip_custom/model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4e1f6a9710dc6279"}},{"code_sha256_prefix":"ddcbd45e940484ee","entry":"gather_features","repo":"scottgeng00/unmet-promise","repo_kind":"official","path":"adapt/clip_custom/loss.py","file_url":"https://github.com/scottgeng00/unmet-promise/blob/HEAD/adapt/clip_custom/loss.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ddcbd45e940484ee"}},{"code_sha256_prefix":"3ccb94c5b63b85a6","entry":"get_features","repo":"scottgeng00/unmet-promise","repo_kind":"official","path":"adapt/src/datasets/common.py","file_url":"https://github.com/scottgeng00/unmet-promise/blob/HEAD/adapt/src/datasets/common.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3ccb94c5b63b85a6"}},{"code_sha256_prefix":"238ff32fcbf410d6","entry":"get_features_helper","repo":"scottgeng00/unmet-promise","repo_kind":"official","path":"adapt/src/datasets/common.py","file_url":"https://github.com/scottgeng00/unmet-promise/blob/HEAD/adapt/src/datasets/common.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"238ff32fcbf410d6"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}