{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/consistency-models-as-a-rich-and-efficient","title":"Consistency Models as a Rich and Efficient Policy Class for Reinforcement Learning","arxiv_id":"2309.16984","date":"2023-09-29","proceeding":null,"authors":["Zihan Ding","Chi Jin"],"abstract":"Score-based generative models like the diffusion model have been testified to be effective in modeling multi-modal data from image generation to reinforcement learning (RL). However, the inference process of diffusion model can be slow, which hinders its usage in RL with iterative sampling. We propose to apply the consistency model as an efficient yet expressive policy representation, namely consistency policy, with an actor-critic style algorithm for three typical RL settings: offline, offline-to-online and online. For offline RL, we demonstrate the expressiveness of generative models as policies from multi-modal data. For offline-to-online RL, the consistency policy is shown to be more computational efficient than diffusion policy, with a comparable performance. For online RL, the consistency policy demonstrates significant speedup and even higher average performances than the diffusion policy.","url_abs":"https://arxiv.org/abs/2309.16984v2","url_pdf":"https://arxiv.org/pdf/2309.16984v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"consistency-models-as-a-rich-and-efficient","repo_url":"https://github.com/quantumiracle/consistency_model_for_reinforcement_learning","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"image-generation","task_name":"Image Generation"},{"task_slug":"offline-rl","task_name":"Offline RL"},{"task_slug":"reinforcement-learning-1","task_name":"Reinforcement Learning (RL)"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2309.16984","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2309.16984"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/quantumiracle/consistency_model_for_reinforcement_learning","reach":null}],"summary":{"ran_fixture":3,"ran_honours":3,"ran":3,"unverified":2},"by_repo_kind":{"official":{"samples":5,"ran":3,"repositories":1},"community":{"samples":5,"ran":5,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":1,"samples":[{"code_sha256_prefix":"465f0a6ce0f5e20e","entry":"append_dims","repo":"Aaditya-Prasad/consistency-policy","repo_kind":"community","path":"consistency_policy/utils.py","file_url":"https://github.com/Aaditya-Prasad/consistency-policy/blob/HEAD/consistency_policy/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":2,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"465f0a6ce0f5e20e"}},{"code_sha256_prefix":"4c60d8998b722013","entry":"linear_schedule","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_honours","verification_level":2,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"4c60d8998b722013"}},{"code_sha256_prefix":"dead3c2b85bf0fc2","entry":"Consistency","repo":"quantumiracle/consistency_model_for_reinforcement_learning","repo_kind":"official","path":"agents/consistency.py","file_url":"https://github.com/quantumiracle/consistency_model_for_reinforcement_learning/blob/HEAD/agents/consistency.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"dead3c2b85bf0fc2"}},{"code_sha256_prefix":"a195e5fbb1ffc8f2","entry":"WeightedL1","repo":"quantumiracle/consistency_model_for_reinforcement_learning","repo_kind":"official","path":"agents/consistency.py","file_url":"https://github.com/quantumiracle/consistency_model_for_reinforcement_learning/blob/HEAD/agents/consistency.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a195e5fbb1ffc8f2"}},{"code_sha256_prefix":"04bd445ebe6af984","entry":"WeightedL2","repo":"quantumiracle/consistency_model_for_reinforcement_learning","repo_kind":"official","path":"agents/consistency.py","file_url":"https://github.com/quantumiracle/consistency_model_for_reinforcement_learning/blob/HEAD/agents/consistency.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"04bd445ebe6af984"}},{"code_sha256_prefix":"eef3fefaf1daa0fb","entry":"euler_to_quat","repo":"Aaditya-Prasad/consistency-policy","repo_kind":"community","path":"consistency_policy/utils.py","file_url":"https://github.com/Aaditya-Prasad/consistency-policy/blob/HEAD/consistency_policy/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"eef3fefaf1daa0fb"}},{"code_sha256_prefix":"f2143ca43f9dc693","entry":"reduce_dims","repo":"Aaditya-Prasad/consistency-policy","repo_kind":"community","path":"consistency_policy/utils.py","file_url":"https://github.com/Aaditya-Prasad/consistency-policy/blob/HEAD/consistency_policy/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f2143ca43f9dc693"}},{"code_sha256_prefix":"0ddc75bcecc6e03f","entry":"sample_n_stratified","repo":"alexander-soare/consistency_policy","repo_kind":"community","path":"consistency_policy/consistency_model.py","file_url":"https://github.com/alexander-soare/consistency_policy/blob/HEAD/consistency_policy/consistency_model.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0ddc75bcecc6e03f"}},{"code_sha256_prefix":"c9631ffcefa55af9","entry":"sample_n_uniform","repo":"alexander-soare/consistency_policy","repo_kind":"community","path":"consistency_policy/consistency_model.py","file_url":"https://github.com/alexander-soare/consistency_policy/blob/HEAD/consistency_policy/consistency_model.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c9631ffcefa55af9"}},{"code_sha256_prefix":"c6104b52f96f7bc9","entry":"WeightedHuber","repo":"quantumiracle/consistency_model_for_reinforcement_learning","repo_kind":"official","path":"agents/consistency.py","file_url":"https://github.com/quantumiracle/consistency_model_for_reinforcement_learning/blob/HEAD/agents/consistency.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c6104b52f96f7bc9"}},{"code_sha256_prefix":"0749c0edc53a9387","entry":"WeightedLoss","repo":"quantumiracle/consistency_model_for_reinforcement_learning","repo_kind":"official","path":"agents/consistency.py","file_url":"https://github.com/quantumiracle/consistency_model_for_reinforcement_learning/blob/HEAD/agents/consistency.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0749c0edc53a9387"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}