{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/training-diffusion-models-with-reinforcement","title":"Training Diffusion Models with Reinforcement Learning","arxiv_id":"2305.13301","date":"2023-05-22","proceeding":null,"authors":["Kevin Black","Michael Janner","Yilun Du","Ilya Kostrikov","Sergey Levine"],"abstract":"Diffusion models are a class of flexible generative models trained with an approximation to the log-likelihood objective. However, most use cases of diffusion models are not concerned with likelihoods, but instead with downstream objectives such as human-perceived image quality or drug effectiveness. In this paper, we investigate reinforcement learning methods for directly optimizing diffusion models for such objectives. We describe how posing denoising as a multi-step decision-making problem enables a class of policy gradient algorithms, which we refer to as denoising diffusion policy optimization (DDPO), that are more effective than alternative reward-weighted likelihood approaches. Empirically, DDPO is able to adapt text-to-image diffusion models to objectives that are difficult to express via prompting, such as image compressibility, and those derived from human feedback, such as aesthetic quality. Finally, we show that DDPO can improve prompt-image alignment using feedback from a vision-language model without the need for additional data collection or human annotation. The project's website can be found at http://rl-diffusion.github.io .","url_abs":"https://arxiv.org/abs/2305.13301v4","url_pdf":"https://arxiv.org/pdf/2305.13301v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"training-diffusion-models-with-reinforcement","repo_url":"https://github.com/kvablack/ddpo-pytorch","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"training-diffusion-models-with-reinforcement","repo_url":"https://github.com/Owen-Oertell/rlcm","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"training-diffusion-models-with-reinforcement","repo_url":"https://github.com/akashsonowal/ddpo-pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"decision-making","task_name":"Decision Making"},{"task_slug":"denoising","task_name":"Denoising"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"reinforcement-learning","task_name":"Reinforcement Learning"},{"task_slug":"text-to-image-generation","task_name":"Text-to-Image Generation"},{"task_slug":"reinforcement-learning-2","task_name":"reinforcement-learning"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/text-to-image-generation-on-drawbench","task":"Text-to-Image Generation","dataset":"DrawBench","model":"LCM (DDPO)","rank_in_archive_order":3,"of":8,"metrics":{"Aesthetics (Laion Aesthtetics Predictor)":"6.0121","Human Preference Alignement (HPSv2)":"0.2803","Text Alignement (SentenceBERT)":"0.5721"},"uses_additional_data":false},{"leaderboard":"/sota/text-to-image-generation-on-drawbench","task":"Text-to-Image Generation","dataset":"DrawBench","model":"Stable Diffusion 1.5 (DDPO)","rank_in_archive_order":6,"of":8,"metrics":{"Aesthetics (Laion Aesthtetics Predictor)":"5.6748","Human Preference Alignement (HPSv2)":"0.2673","Text Alignement (SentenceBERT)":"0.6024"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2305.13301","atlas_url":"https://app.syntology.ai/?focus=2305.13301","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.13301"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kvablack/ddpo-pytorch","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Owen-Oertell/rlcm","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/akashsonowal/ddpo-pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":4,"unverified":2},"by_repo_kind":{"official":{"samples":1,"ran":0,"repositories":1},"listed":{"samples":5,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"839da88e68e0677c","entry":"aesthetic_model_normalize","repo":"akashsonowal/ddpo-pytorch","repo_kind":"listed","path":"ddpo_pytorch/aesthetic_scorer.py","file_url":"https://github.com/akashsonowal/ddpo-pytorch/blob/HEAD/ddpo_pytorch/aesthetic_scorer.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"839da88e68e0677c"}},{"code_sha256_prefix":"b4f4d87568dc7dbf","entry":"aesthetic_scoring","repo":"akashsonowal/ddpo-pytorch","repo_kind":"listed","path":"ddpo_pytorch/aesthetic_scorer.py","file_url":"https://github.com/akashsonowal/ddpo-pytorch/blob/HEAD/ddpo_pytorch/aesthetic_scorer.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b4f4d87568dc7dbf"}},{"code_sha256_prefix":"99b4cfbc8f8c70db","entry":"calculate_log_probs","repo":"akashsonowal/ddpo-pytorch","repo_kind":"listed","path":"ddpo_pytorch/utils.py","file_url":"https://github.com/akashsonowal/ddpo-pytorch/blob/HEAD/ddpo_pytorch/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"99b4cfbc8f8c70db"}},{"code_sha256_prefix":"ae64c699ad5628ca","entry":"decoding_fn","repo":"akashsonowal/ddpo-pytorch","repo_kind":"listed","path":"ddpo_pytorch/utils.py","file_url":"https://github.com/akashsonowal/ddpo-pytorch/blob/HEAD/ddpo_pytorch/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ae64c699ad5628ca"}},{"code_sha256_prefix":"b6e22297f43ef430","entry":"get_config","repo":"kvablack/ddpo-pytorch","repo_kind":"official","path":"config/dgx.py","file_url":"https://github.com/kvablack/ddpo-pytorch/blob/HEAD/config/dgx.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b6e22297f43ef430"}},{"code_sha256_prefix":"ec9dd7a0999c059f","entry":"load_aesthetic_model_weights","repo":"akashsonowal/ddpo-pytorch","repo_kind":"listed","path":"ddpo_pytorch/aesthetic_scorer.py","file_url":"https://github.com/akashsonowal/ddpo-pytorch/blob/HEAD/ddpo_pytorch/aesthetic_scorer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ec9dd7a0999c059f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}