{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/diffharmony-latent-diffusion-model-meets","title":"DiffHarmony: Latent Diffusion Model Meets Image Harmonization","arxiv_id":"2404.06139","date":"2024-04-09","proceeding":null,"authors":["Pengfei Zhou","Fangxiang Feng","Xiaojie Wang"],"abstract":"Image harmonization, which involves adjusting the foreground of a composite image to attain a unified visual consistency with the background, can be conceptualized as an image-to-image translation task. Diffusion models have recently promoted the rapid development of image-to-image translation tasks . However, training diffusion models from scratch is computationally intensive. Fine-tuning pre-trained latent diffusion models entails dealing with the reconstruction error induced by the image compression autoencoder, making it unsuitable for image generation tasks that involve pixel-level evaluation metrics. To deal with these issues, in this paper, we first adapt a pre-trained latent diffusion model to the image harmonization task to generate the harmonious but potentially blurry initial images. Then we implement two strategies: utilizing higher-resolution images during inference and incorporating an additional refinement stage, to further enhance the clarity of the initially harmonized images. Extensive experiments on iHarmony4 datasets demonstrate the superiority of our proposed method. The code and model will be made publicly available at https://github.com/nicecv/DiffHarmony .","url_abs":"https://arxiv.org/abs/2404.06139v1","url_pdf":"https://arxiv.org/pdf/2404.06139v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"diffharmony-latent-diffusion-model-meets","repo_url":"https://github.com/nicecv/diffharmony","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"image-compression","task_name":"Image Compression"},{"task_slug":"image-generation","task_name":"Image Generation"},{"task_slug":"image-harmonization","task_name":"Image Harmonization"},{"task_slug":"image-to-image-translation","task_name":"Image-to-Image Translation"},{"task_slug":"translation","task_name":"Translation"},{"task_slug":"model","task_name":"model"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"},{"method_slug":"latent-diffusion-model","method_name":"Latent Diffusion Model"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2404.06139","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.06139"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/nicecv/diffharmony","reach":{"status":"ok"}}],"summary":{"ran":7,"unverified":3},"by_repo_kind":{"official":{"samples":10,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":10,"samples":[{"code_sha256_prefix":"63180d57531be192","entry":"extract_ds_name","repo":"nicecv/diffharmony","repo_kind":"official","path":"src/dataset/ihd_dataset.py","file_url":"https://github.com/nicecv/diffharmony/blob/HEAD/src/dataset/ihd_dataset.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"63180d57531be192"}},{"code_sha256_prefix":"f2e5b96191ed112e","entry":"get_paths","repo":"nicecv/diffharmony","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/nicecv/diffharmony/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f2e5b96191ed112e"}},{"code_sha256_prefix":"e00c6f7b7efc9499","entry":"get_paths","repo":"nicecv/diffharmony","repo_kind":"official","path":"src/dataset/ihd_dataset.py","file_url":"https://github.com/nicecv/diffharmony/blob/HEAD/src/dataset/ihd_dataset.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e00c6f7b7efc9499"}},{"code_sha256_prefix":"dba6fe6b30a3db16","entry":"harm_to_names","repo":"nicecv/diffharmony","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/nicecv/diffharmony/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"dba6fe6b30a3db16"}},{"code_sha256_prefix":"cac844d58d7cd2d7","entry":"prepare_mask_and_masked_image","repo":"nicecv/diffharmony","repo_kind":"official","path":"src/pipelines/pipeline_stable_diffusion_harmony.py","file_url":"https://github.com/nicecv/diffharmony/blob/HEAD/src/pipelines/pipeline_stable_diffusion_harmony.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"cac844d58d7cd2d7"}},{"code_sha256_prefix":"9178a3aaaf1a46b7","entry":"read_jsonl_file","repo":"nicecv/diffharmony","repo_kind":"official","path":"src/dataset/ihd_dataset.py","file_url":"https://github.com/nicecv/diffharmony/blob/HEAD/src/dataset/ihd_dataset.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9178a3aaaf1a46b7"}},{"code_sha256_prefix":"4aa2c69430a0613f","entry":"tensor_to_pil","repo":"nicecv/diffharmony","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/nicecv/diffharmony/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4aa2c69430a0613f"}},{"code_sha256_prefix":"acf6f9a764cebc1a","entry":"get_down_block","repo":"nicecv/diffharmony","repo_kind":"official","path":"src/models/unet_2d_blocks.py","file_url":"https://github.com/nicecv/diffharmony/blob/HEAD/src/models/unet_2d_blocks.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"acf6f9a764cebc1a"}},{"code_sha256_prefix":"bfc9bbd91d301bce","entry":"get_up_block","repo":"nicecv/diffharmony","repo_kind":"official","path":"src/models/unet_2d_blocks.py","file_url":"https://github.com/nicecv/diffharmony/blob/HEAD/src/models/unet_2d_blocks.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bfc9bbd91d301bce"}},{"code_sha256_prefix":"d81381b327c76edd","entry":"zero_module","repo":"nicecv/diffharmony","repo_kind":"official","path":"src/models/vae.py","file_url":"https://github.com/nicecv/diffharmony/blob/HEAD/src/models/vae.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d81381b327c76edd"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}