{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/noisecollage-a-layout-aware-text-to-image","title":"NoiseCollage: A Layout-Aware Text-to-Image Diffusion Model Based on Noise Cropping and Merging","arxiv_id":"2403.03485","date":"2024-03-06","proceeding":"CVPR 2024 1","authors":["Takahiro Shirakawa","Seiichi Uchida"],"abstract":"Layout-aware text-to-image generation is a task to generate multi-object images that reflect layout conditions in addition to text conditions. The current layout-aware text-to-image diffusion models still have several issues, including mismatches between the text and layout conditions and quality degradation of generated images. This paper proposes a novel layout-aware text-to-image diffusion model called NoiseCollage to tackle these issues. During the denoising process, NoiseCollage independently estimates noises for individual objects and then crops and merges them into a single noise. This operation helps avoid condition mismatches; in other words, it can put the right objects in the right places. Qualitative and quantitative evaluations show that NoiseCollage outperforms several state-of-the-art models. These successful results indicate that the crop-and-merge operation of noises is a reasonable strategy to control image generation. We also show that NoiseCollage can be integrated with ControlNet to use edges, sketches, and pose skeletons as additional conditions. Experimental results show that this integration boosts the layout accuracy of ControlNet. The code is available at https://github.com/univ-esuty/noisecollage.","url_abs":"https://arxiv.org/abs/2403.03485v1","url_pdf":"https://arxiv.org/pdf/2403.03485v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"noisecollage-a-layout-aware-text-to-image","repo_url":"https://github.com/univ-esuty/noisecollage","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"jax","reach":{"status":"ok"}}],"tasks":[{"task_slug":"denoising","task_name":"Denoising"},{"task_slug":"image-generation","task_name":"Image Generation"},{"task_slug":"text-to-image-generation-1","task_name":"Text to Image Generation"},{"task_slug":"text-to-image-generation","task_name":"Text-to-Image Generation"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2403.03485","atlas_url":"https://app.syntology.ai/?focus=2403.03485","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.03485"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/univ-esuty/noisecollage","reach":{"status":"ok"}}],"summary":{"ran":4,"ran_draft_wrong":1,"unverified":4},"by_repo_kind":{"official":{"samples":9,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":9,"samples":[{"code_sha256_prefix":"01ff60f53da84692","entry":"batch_per_cosine_similarity","repo":"univ-esuty/noisecollage","repo_kind":"official","path":"evaluation/model.py","file_url":"https://github.com/univ-esuty/noisecollage/blob/HEAD/evaluation/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"01ff60f53da84692"}},{"code_sha256_prefix":"1bd79f01f560a239","entry":"batch_per_mse","repo":"univ-esuty/noisecollage","repo_kind":"official","path":"evaluation/model.py","file_url":"https://github.com/univ-esuty/noisecollage/blob/HEAD/evaluation/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1bd79f01f560a239"}},{"code_sha256_prefix":"06c26d7b1073325f","entry":"eval_result","repo":"univ-esuty/noisecollage","repo_kind":"official","path":"gen_img_by_noisecollage-with-controlnet.py","file_url":"https://github.com/univ-esuty/noisecollage/blob/HEAD/gen_img_by_noisecollage-with-controlnet.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"06c26d7b1073325f"}},{"code_sha256_prefix":"125f7ee5f275ed2a","entry":"get_activation","repo":"univ-esuty/noisecollage","repo_kind":"official","path":"diffusers/models/activations.py","file_url":"https://github.com/univ-esuty/noisecollage/blob/HEAD/diffusers/models/activations.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"125f7ee5f275ed2a"}},{"code_sha256_prefix":"bea2d776a332f2b0","entry":"rescale_noise_cfg","repo":"univ-esuty/noisecollage","repo_kind":"official","path":"pipeline_custom/pipeline_noise_collage_controlnet.py","file_url":"https://github.com/univ-esuty/noisecollage/blob/HEAD/pipeline_custom/pipeline_noise_collage_controlnet.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bea2d776a332f2b0"}},{"code_sha256_prefix":"edf5d54bc7a50c95","entry":"mse","repo":"univ-esuty/noisecollage","repo_kind":"official","path":"evaluation/model.py","file_url":"https://github.com/univ-esuty/noisecollage/blob/HEAD/evaluation/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"edf5d54bc7a50c95"}},{"code_sha256_prefix":"f4ac91315a94774b","entry":"text_encoder_attn_modules","repo":"univ-esuty/noisecollage","repo_kind":"official","path":"diffusers/loaders.py","file_url":"https://github.com/univ-esuty/noisecollage/blob/HEAD/diffusers/loaders.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f4ac91315a94774b"}},{"code_sha256_prefix":"2c476d130bd89887","entry":"text_encoder_lora_state_dict","repo":"univ-esuty/noisecollage","repo_kind":"official","path":"diffusers/loaders.py","file_url":"https://github.com/univ-esuty/noisecollage/blob/HEAD/diffusers/loaders.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2c476d130bd89887"}},{"code_sha256_prefix":"ccca296c90beae91","entry":"text_encoder_mlp_modules","repo":"univ-esuty/noisecollage","repo_kind":"official","path":"diffusers/loaders.py","file_url":"https://github.com/univ-esuty/noisecollage/blob/HEAD/diffusers/loaders.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ccca296c90beae91"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}