{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/instancediffusion-instance-level-control-for","title":"InstanceDiffusion: Instance-level Control for Image Generation","arxiv_id":"2402.03290","date":"2024-02-05","proceeding":"CVPR 2024 1","authors":["Xudong Wang","Trevor Darrell","Sai Saketh Rambhatla","Rohit Girdhar","Ishan Misra"],"abstract":"Text-to-image diffusion models produce high quality images but do not offer control over individual instances in the image. We introduce InstanceDiffusion that adds precise instance-level control to text-to-image diffusion models. InstanceDiffusion supports free-form language conditions per instance and allows flexible ways to specify instance locations such as simple single points, scribbles, bounding boxes or intricate instance segmentation masks, and combinations thereof. We propose three major changes to text-to-image models that enable precise instance-level control. Our UniFusion block enables instance-level conditions for text-to-image models, the ScaleU block improves image fidelity, and our Multi-instance Sampler improves generations for multiple instances. InstanceDiffusion significantly surpasses specialized state-of-the-art models for each location condition. Notably, on the COCO dataset, we outperform previous state-of-the-art by 20.4% AP$_{50}^\\text{box}$ for box inputs, and 25.4% IoU for mask inputs.","url_abs":"https://arxiv.org/abs/2402.03290v1","url_pdf":"https://arxiv.org/pdf/2402.03290v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"instancediffusion-instance-level-control-for","repo_url":"https://github.com/frank-xwang/InstanceDiffusion","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"conditional-text-to-image-synthesis","task_name":"Conditional Text-to-Image Synthesis"},{"task_slug":"image-generation","task_name":"Image Generation"},{"task_slug":"instance-segmentation","task_name":"Instance Segmentation"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/conditional-text-to-image-synthesis-on-coco-1","task":"Conditional Text-to-Image Synthesis","dataset":"COCO-MIG","model":"Instance Diffusion (zero-shot)","rank_in_archive_order":3,"of":5,"metrics":{"instance success rate":"0.51","mIoU":"0.46"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2402.03290","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.03290"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/frank-xwang/InstanceDiffusion","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":17,"unverified":1},"by_repo_kind":{"official":{"samples":18,"ran":17,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"6a31c83c41f11b71","entry":"batch_to_device","repo":"frank-xwang/InstanceDiffusion","repo_kind":"official","path":"dataset/jsondataset.py","file_url":"https://github.com/frank-xwang/InstanceDiffusion/blob/HEAD/dataset/jsondataset.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6a31c83c41f11b71"}},{"code_sha256_prefix":"24680b34da633bbd","entry":"close_contour","repo":"frank-xwang/InstanceDiffusion","repo_kind":"official","path":"dataset/decode_item.py","file_url":"https://github.com/frank-xwang/InstanceDiffusion/blob/HEAD/dataset/decode_item.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"24680b34da633bbd"}},{"code_sha256_prefix":"2fac02c349935452","entry":"complete_mask","repo":"frank-xwang/InstanceDiffusion","repo_kind":"official","path":"inference.py","file_url":"https://github.com/frank-xwang/InstanceDiffusion/blob/HEAD/inference.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2fac02c349935452"}},{"code_sha256_prefix":"0226083d79fbec56","entry":"convert_coco_box","repo":"frank-xwang/InstanceDiffusion","repo_kind":"official","path":"eval_local.py","file_url":"https://github.com/frank-xwang/InstanceDiffusion/blob/HEAD/eval_local.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0226083d79fbec56"}},{"code_sha256_prefix":"8336fe993ab97e1e","entry":"create_ema","repo":"frank-xwang/InstanceDiffusion","repo_kind":"official","path":"utils/model.py","file_url":"https://github.com/frank-xwang/InstanceDiffusion/blob/HEAD/utils/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8336fe993ab97e1e"}},{"code_sha256_prefix":"0ededb39dd4d5702","entry":"decode_base64_to_pillow","repo":"frank-xwang/InstanceDiffusion","repo_kind":"official","path":"dataset/decode_item.py","file_url":"https://github.com/frank-xwang/InstanceDiffusion/blob/HEAD/dataset/decode_item.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0ededb39dd4d5702"}},{"code_sha256_prefix":"ad34702f4726aa2e","entry":"decode_tensor_from_string","repo":"frank-xwang/InstanceDiffusion","repo_kind":"official","path":"dataset/decode_item.py","file_url":"https://github.com/frank-xwang/InstanceDiffusion/blob/HEAD/dataset/decode_item.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ad34702f4726aa2e"}},{"code_sha256_prefix":"8191ff51c4b074ff","entry":"draw_box","repo":"frank-xwang/InstanceDiffusion","repo_kind":"official","path":"dataset/base_dataset.py","file_url":"https://github.com/frank-xwang/InstanceDiffusion/blob/HEAD/dataset/base_dataset.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8191ff51c4b074ff"}},{"code_sha256_prefix":"6de5057cb6ed75b0","entry":"encode_tensor_as_string","repo":"frank-xwang/InstanceDiffusion","repo_kind":"official","path":"dataset-generation/submitit_automatic_label_ram_save_json.py","file_url":"https://github.com/frank-xwang/InstanceDiffusion/blob/HEAD/dataset-generation/submitit_automatic_label_ram_save_json.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6de5057cb6ed75b0"}},{"code_sha256_prefix":"0d1f144ac0ae3d6b","entry":"get_clip_feature_text","repo":"frank-xwang/InstanceDiffusion","repo_kind":"official","path":"dataset-generation/submitit_automatic_label_ram_save_json.py","file_url":"https://github.com/frank-xwang/InstanceDiffusion/blob/HEAD/dataset-generation/submitit_automatic_label_ram_save_json.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0d1f144ac0ae3d6b"}},{"code_sha256_prefix":"dbb0b3758a575e0e","entry":"get_point_from_box","repo":"frank-xwang/InstanceDiffusion","repo_kind":"official","path":"eval_local.py","file_url":"https://github.com/frank-xwang/InstanceDiffusion/blob/HEAD/eval_local.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"dbb0b3758a575e0e"}},{"code_sha256_prefix":"cc40c350f2fdbdf0","entry":"preprocess_text","repo":"frank-xwang/InstanceDiffusion","repo_kind":"official","path":"dataset-generation/submitit_automatic_label_ram_save_json.py","file_url":"https://github.com/frank-xwang/InstanceDiffusion/blob/HEAD/dataset-generation/submitit_automatic_label_ram_save_json.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cc40c350f2fdbdf0"}},{"code_sha256_prefix":"1b3923351aeaafb3","entry":"rescale_box","repo":"frank-xwang/InstanceDiffusion","repo_kind":"official","path":"inference.py","file_url":"https://github.com/frank-xwang/InstanceDiffusion/blob/HEAD/inference.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1b3923351aeaafb3"}},{"code_sha256_prefix":"075b026765e25e3e","entry":"rescale_points","repo":"frank-xwang/InstanceDiffusion","repo_kind":"official","path":"eval_local.py","file_url":"https://github.com/frank-xwang/InstanceDiffusion/blob/HEAD/eval_local.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"075b026765e25e3e"}},{"code_sha256_prefix":"c9b050c82d78f83d","entry":"sub_batch","repo":"frank-xwang/InstanceDiffusion","repo_kind":"official","path":"dataset/jsondataset.py","file_url":"https://github.com/frank-xwang/InstanceDiffusion/blob/HEAD/dataset/jsondataset.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c9b050c82d78f83d"}},{"code_sha256_prefix":"08fa56c4d4f18a91","entry":"to_valid","repo":"frank-xwang/InstanceDiffusion","repo_kind":"official","path":"dataset/base_dataset.py","file_url":"https://github.com/frank-xwang/InstanceDiffusion/blob/HEAD/dataset/base_dataset.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"08fa56c4d4f18a91"}},{"code_sha256_prefix":"ce28a1eb6ea4fde7","entry":"to_valid_scribbles","repo":"frank-xwang/InstanceDiffusion","repo_kind":"official","path":"dataset/base_dataset.py","file_url":"https://github.com/frank-xwang/InstanceDiffusion/blob/HEAD/dataset/base_dataset.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ce28a1eb6ea4fde7"}},{"code_sha256_prefix":"37960ce701410505","entry":"load_ckpt","repo":"frank-xwang/InstanceDiffusion","repo_kind":"official","path":"utils/model.py","file_url":"https://github.com/frank-xwang/InstanceDiffusion/blob/HEAD/utils/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"37960ce701410505"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}