{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/steered-diffusion-a-generalized-framework-for-1","title":"Steered Diffusion: A Generalized Framework for Plug-and-Play Conditional Image Synthesis","arxiv_id":"2310.00224","date":"2023-09-30","proceeding":"ICCV 2023 1","authors":["Nithin Gopalakrishnan Nair","Anoop Cherian","Suhas Lohit","Ye Wang","Toshiaki Koike-Akino","Vishal M. Patel","Tim K. Marks"],"abstract":"Conditional generative models typically demand large annotated training sets to achieve high-quality synthesis. As a result, there has been significant interest in designing models that perform plug-and-play generation, i.e., to use a predefined or pretrained model, which is not explicitly trained on the generative task, to guide the generative process (e.g., using language). However, such guidance is typically useful only towards synthesizing high-level semantics rather than editing fine-grained details as in image-to-image translation tasks. To this end, and capitalizing on the powerful fine-grained generative control offered by the recent diffusion-based generative models, we introduce Steered Diffusion, a generalized framework for photorealistic zero-shot conditional image generation using a diffusion model trained for unconditional generation. The key idea is to steer the image generation of the diffusion model at inference time via designing a loss using a pre-trained inverse model that characterizes the conditional task. This loss modulates the sampling trajectory of the diffusion process. Our framework allows for easy incorporation of multiple conditions during inference. We present experiments using steered diffusion on several tasks including inpainting, colorization, text-guided semantic editing, and image super-resolution. Our results demonstrate clear qualitative and quantitative improvements over state-of-the-art diffusion-based plug-and-play models while adding negligible additional computational cost.","url_abs":"https://arxiv.org/abs/2310.00224v1","url_pdf":"https://arxiv.org/pdf/2310.00224v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"steered-diffusion-a-generalized-framework-for-1","repo_url":"https://github.com/merlresearch/SteeredDiffusion","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"AGPL-3.0"}}],"tasks":[{"task_slug":"colorization","task_name":"Colorization"},{"task_slug":"conditional-image-generation","task_name":"Conditional Image Generation"},{"task_slug":"image-generation","task_name":"Image Generation"},{"task_slug":"image-super-resolution","task_name":"Image Super-Resolution"},{"task_slug":"image-to-image-translation","task_name":"Image-to-Image Translation"},{"task_slug":"super-resolution","task_name":"Super-Resolution"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2310.00224","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.00224"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/merlresearch/SteeredDiffusion","reach":{"status":"ok","spdx":"AGPL-3.0"}}],"summary":{"ran":3,"unverified":7},"by_repo_kind":{"official":{"samples":10,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":10,"samples":[{"code_sha256_prefix":"b8dcee3ca1d39c22","entry":"find_cossim","repo":"merlresearch/SteeredDiffusion","repo_kind":"official","path":"networks/FARL_losses/farl_losses.py","file_url":"https://github.com/merlresearch/SteeredDiffusion/blob/HEAD/networks/FARL_losses/farl_losses.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"b8dcee3ca1d39c22"}},{"code_sha256_prefix":"02743b0d62e52587","entry":"ssim","repo":"merlresearch/SteeredDiffusion","repo_kind":"official","path":"losses/ssim.py","file_url":"https://github.com/merlresearch/SteeredDiffusion/blob/HEAD/losses/ssim.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"02743b0d62e52587"}},{"code_sha256_prefix":"eb3584d528a35911","entry":"support_sz","repo":"merlresearch/SteeredDiffusion","repo_kind":"official","path":"guided_diffusion/guided_diffusion/interp_methods.py","file_url":"https://github.com/merlresearch/SteeredDiffusion/blob/HEAD/guided_diffusion/guided_diffusion/interp_methods.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"eb3584d528a35911"}},{"code_sha256_prefix":"8e6dfffd626f0962","entry":"create_window","repo":"merlresearch/SteeredDiffusion","repo_kind":"official","path":"losses/ssim.py","file_url":"https://github.com/merlresearch/SteeredDiffusion/blob/HEAD/losses/ssim.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"8e6dfffd626f0962"}},{"code_sha256_prefix":"69f91d76e488739d","entry":"cubic","repo":"merlresearch/SteeredDiffusion","repo_kind":"official","path":"guided_diffusion/guided_diffusion/interp_methods.py","file_url":"https://github.com/merlresearch/SteeredDiffusion/blob/HEAD/guided_diffusion/guided_diffusion/interp_methods.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"69f91d76e488739d"}},{"code_sha256_prefix":"d98b7747584eccdf","entry":"d_clip_loss","repo":"merlresearch/SteeredDiffusion","repo_kind":"official","path":"networks/FARL_losses/farl_losses.py","file_url":"https://github.com/merlresearch/SteeredDiffusion/blob/HEAD/networks/FARL_losses/farl_losses.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"d98b7747584eccdf"}},{"code_sha256_prefix":"b5064d35fa99da32","entry":"fetch_file_cached","repo":"merlresearch/SteeredDiffusion","repo_kind":"official","path":"utils/download_models_func.py","file_url":"https://github.com/merlresearch/SteeredDiffusion/blob/HEAD/utils/download_models_func.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"b5064d35fa99da32"}},{"code_sha256_prefix":"e2424ce033ba3a0f","entry":"gaussian","repo":"merlresearch/SteeredDiffusion","repo_kind":"official","path":"losses/ssim.py","file_url":"https://github.com/merlresearch/SteeredDiffusion/blob/HEAD/losses/ssim.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"e2424ce033ba3a0f"}},{"code_sha256_prefix":"135b3dc835ffe6ad","entry":"get_confirm_token","repo":"merlresearch/SteeredDiffusion","repo_kind":"official","path":"utils/download_models_func.py","file_url":"https://github.com/merlresearch/SteeredDiffusion/blob/HEAD/utils/download_models_func.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"135b3dc835ffe6ad"}},{"code_sha256_prefix":"ae962b80c12bc789","entry":"set_framework_dependencies","repo":"merlresearch/SteeredDiffusion","repo_kind":"official","path":"guided_diffusion/guided_diffusion/interp_methods.py","file_url":"https://github.com/merlresearch/SteeredDiffusion/blob/HEAD/guided_diffusion/guided_diffusion/interp_methods.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"ae962b80c12bc789"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}