{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/average-constrained-policy-optimization","title":"ACPO: A Policy Optimization Algorithm for Average MDPs with Constraints","arxiv_id":"2302.00808","date":"2023-02-02","proceeding":null,"authors":["Akhil Agnihotri","Rahul Jain","Haipeng Luo"],"abstract":"Reinforcement Learning (RL) for constrained MDPs (CMDPs) is an increasingly important problem for various applications. Often, the average criterion is more suitable than the discounted criterion. Yet, RL for average-CMDPs (ACMDPs) remains a challenging problem. Algorithms designed for discounted constrained RL problems often do not perform well for the average CMDP setting. In this paper, we introduce a new policy optimization with function approximation algorithm for constrained MDPs with the average criterion. The Average-Constrained Policy Optimization (ACPO) algorithm is inspired by trust region-based policy optimization algorithms. We develop basic sensitivity theory for average CMDPs, and then use the corresponding bounds in the design of the algorithm. We provide theoretical guarantees on its performance, and through extensive experimental work in various challenging OpenAI Gym environments, show its superior empirical performance when compared to other state-of-the-art algorithms adapted for the ACMDPs.","url_abs":"https://arxiv.org/abs/2302.00808v4","url_pdf":"https://arxiv.org/pdf/2302.00808v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[],"tasks":[{"task_slug":"openai-gym","task_name":"OpenAI Gym"},{"task_slug":"reinforcement-learning-1","task_name":"Reinforcement Learning (RL)"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2302.00808","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2302.00808"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/openai/safety-starter-agents","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/rll/rllab","reach":{"status":"ok","spdx":"NOASSERTION"}}],"summary":{"ran":2,"ran_draft_wrong":1,"unverified":9},"by_repo_kind":{"found_in_text":{"samples":12,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"7e36f1c357c4618d","entry":"colorize","repo":"openai/safety-starter-agents","repo_kind":"found_in_text","path":"safe_rl/utils/logx.py","file_url":"https://github.com/openai/safety-starter-agents/blob/HEAD/safe_rl/utils/logx.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7e36f1c357c4618d"}},{"code_sha256_prefix":"57542f87e248db45","entry":"combined_shape","repo":"openai/safety-starter-agents","repo_kind":"found_in_text","path":"safe_rl/pg/utils.py","file_url":"https://github.com/openai/safety-starter-agents/blob/HEAD/safe_rl/pg/utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"57542f87e248db45"}},{"code_sha256_prefix":"f29202d8108bccd3","entry":"flat_concat","repo":"openai/safety-starter-agents","repo_kind":"found_in_text","path":"safe_rl/pg/trust_region.py","file_url":"https://github.com/openai/safety-starter-agents/blob/HEAD/safe_rl/pg/trust_region.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f29202d8108bccd3"}},{"code_sha256_prefix":"042875cf711accd0","entry":"assign_params_from_flat","repo":"openai/safety-starter-agents","repo_kind":"found_in_text","path":"safe_rl/utils/mpi_tf.py","file_url":"https://github.com/openai/safety-starter-agents/blob/HEAD/safe_rl/utils/mpi_tf.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"042875cf711accd0"}},{"code_sha256_prefix":"331e2222a5374f81","entry":"flat_grad","repo":"openai/safety-starter-agents","repo_kind":"found_in_text","path":"safe_rl/pg/trust_region.py","file_url":"https://github.com/openai/safety-starter-agents/blob/HEAD/safe_rl/pg/trust_region.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"331e2222a5374f81"}},{"code_sha256_prefix":"a2580d7c4a763348","entry":"get_vars","repo":"openai/safety-starter-agents","repo_kind":"found_in_text","path":"safe_rl/sac/sac.py","file_url":"https://github.com/openai/safety-starter-agents/blob/HEAD/safe_rl/sac/sac.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a2580d7c4a763348"}},{"code_sha256_prefix":"c884bc43c5209da0","entry":"hessian_vector_product","repo":"openai/safety-starter-agents","repo_kind":"found_in_text","path":"safe_rl/pg/trust_region.py","file_url":"https://github.com/openai/safety-starter-agents/blob/HEAD/safe_rl/pg/trust_region.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c884bc43c5209da0"}},{"code_sha256_prefix":"3e9d35cf75437bdf","entry":"keys_as_sorted_list","repo":"openai/safety-starter-agents","repo_kind":"found_in_text","path":"safe_rl/pg/utils.py","file_url":"https://github.com/openai/safety-starter-agents/blob/HEAD/safe_rl/pg/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3e9d35cf75437bdf"}},{"code_sha256_prefix":"e68b07bcdded00fa","entry":"mlp","repo":"openai/safety-starter-agents","repo_kind":"found_in_text","path":"safe_rl/pg/network.py","file_url":"https://github.com/openai/safety-starter-agents/blob/HEAD/safe_rl/pg/network.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e68b07bcdded00fa"}},{"code_sha256_prefix":"17b03c462ce92576","entry":"placeholder","repo":"openai/safety-starter-agents","repo_kind":"found_in_text","path":"safe_rl/sac/sac.py","file_url":"https://github.com/openai/safety-starter-agents/blob/HEAD/safe_rl/sac/sac.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"17b03c462ce92576"}},{"code_sha256_prefix":"006604b1b94a9fe7","entry":"restore_tf_graph","repo":"openai/safety-starter-agents","repo_kind":"found_in_text","path":"safe_rl/utils/logx.py","file_url":"https://github.com/openai/safety-starter-agents/blob/HEAD/safe_rl/utils/logx.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"006604b1b94a9fe7"}},{"code_sha256_prefix":"81aea179a4407ed7","entry":"values_as_sorted_list","repo":"openai/safety-starter-agents","repo_kind":"found_in_text","path":"safe_rl/pg/utils.py","file_url":"https://github.com/openai/safety-starter-agents/blob/HEAD/safe_rl/pg/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"81aea179a4407ed7"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}