{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/tackling-structural-hallucination-in-image","title":"Tackling Structural Hallucination in Image Translation with Local Diffusion","arxiv_id":"2404.05980","date":"2024-04-09","proceeding":null,"authors":["Seunghoi Kim","Chen Jin","Tom Diethe","Matteo Figini","Henry F. J. Tregidgo","Asher Mullokandov","Philip Teare","Daniel C. Alexander"],"abstract":"Recent developments in diffusion models have advanced conditioned image generation, yet they struggle with reconstructing out-of-distribution (OOD) images, such as unseen tumors in medical images, causing \"image hallucination\" and risking misdiagnosis. We hypothesize such hallucinations result from local OOD regions in the conditional images. We verify that partitioning the OOD region and conducting separate image generations alleviates hallucinations in several applications. From this, we propose a training-free diffusion framework that reduces hallucination with multiple Local Diffusion processes. Our approach involves OOD estimation followed by two modules: a \"branching\" module generates locally both within and outside OOD regions, and a \"fusion\" module integrates these predictions into one. Our evaluation shows our method mitigates hallucination over baseline models quantitatively and qualitatively, reducing misdiagnosis by 40% and 25% in the real-world medical and natural image datasets, respectively. It also demonstrates compatibility with various pre-trained diffusion models.","url_abs":"https://arxiv.org/abs/2404.05980v5","url_pdf":"https://arxiv.org/pdf/2404.05980v5.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"tackling-structural-hallucination-in-image","repo_url":"https://github.com/edshkim98/localdiffusion-hallucination","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"hallucination","task_name":"Hallucination"},{"task_slug":"image-generation","task_name":"Image Generation"},{"task_slug":"translation","task_name":"Translation"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2404.05980","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.05980"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/edshkim98/localdiffusion-hallucination","reach":{"status":"ok"}}],"summary":{"ran":5,"ran_violates":4,"ran_draft_wrong":1},"by_repo_kind":{"official":{"samples":10,"ran":10,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":10,"samples":[{"code_sha256_prefix":"a47706b35198fab3","entry":"convert_image_to","repo":"edshkim98/localdiffusion-hallucination","repo_kind":"official","path":"data.py","file_url":"https://github.com/edshkim98/localdiffusion-hallucination/blob/HEAD/data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a47706b35198fab3"}},{"code_sha256_prefix":"d1ef6b8cb9a28a53","entry":"default","repo":"edshkim98/localdiffusion-hallucination","repo_kind":"official","path":"ddpm.py","file_url":"https://github.com/edshkim98/localdiffusion-hallucination/blob/HEAD/ddpm.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d1ef6b8cb9a28a53"}},{"code_sha256_prefix":"aa5486a3650902d8","entry":"exists","repo":"edshkim98/localdiffusion-hallucination","repo_kind":"official","path":"attend.py","file_url":"https://github.com/edshkim98/localdiffusion-hallucination/blob/HEAD/attend.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"aa5486a3650902d8"}},{"code_sha256_prefix":"608e364a9d2376a3","entry":"exists","repo":"edshkim98/localdiffusion-hallucination","repo_kind":"official","path":"ddpm.py","file_url":"https://github.com/edshkim98/localdiffusion-hallucination/blob/HEAD/ddpm.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"608e364a9d2376a3"}},{"code_sha256_prefix":"1f8ce62a1ff89683","entry":"load_config","repo":"edshkim98/localdiffusion-hallucination","repo_kind":"official","path":"utils.py","file_url":"https://github.com/edshkim98/localdiffusion-hallucination/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1f8ce62a1ff89683"}},{"code_sha256_prefix":"d68f986c1818d923","entry":"my_collate","repo":"edshkim98/localdiffusion-hallucination","repo_kind":"official","path":"data.py","file_url":"https://github.com/edshkim98/localdiffusion-hallucination/blob/HEAD/data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d68f986c1818d923"}},{"code_sha256_prefix":"52e10a7d6b0fa036","entry":"once","repo":"edshkim98/localdiffusion-hallucination","repo_kind":"official","path":"attend.py","file_url":"https://github.com/edshkim98/localdiffusion-hallucination/blob/HEAD/attend.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"52e10a7d6b0fa036"}},{"code_sha256_prefix":"8cd8e6547ade35f4","entry":"right_pad_dims_to","repo":"edshkim98/localdiffusion-hallucination","repo_kind":"official","path":"ddpm.py","file_url":"https://github.com/edshkim98/localdiffusion-hallucination/blob/HEAD/ddpm.py","link_basis":"plan_row","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8cd8e6547ade35f4"}},{"code_sha256_prefix":"2465d55c0e5333c1","entry":"set_min_max_val","repo":"edshkim98/localdiffusion-hallucination","repo_kind":"official","path":"utils.py","file_url":"https://github.com/edshkim98/localdiffusion-hallucination/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2465d55c0e5333c1"}},{"code_sha256_prefix":"9b85c24defa296b7","entry":"weighted_bce","repo":"edshkim98/localdiffusion-hallucination","repo_kind":"official","path":"train_seg.py","file_url":"https://github.com/edshkim98/localdiffusion-hallucination/blob/HEAD/train_seg.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9b85c24defa296b7"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}