{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/competitive-policy-optimization","title":"Competitive Policy Optimization","arxiv_id":"2006.10611","date":"2020-06-18","proceeding":null,"authors":["Manish Prajapat","Kamyar Azizzadenesheli","Alexander Liniger","Yisong Yue","Anima Anandkumar"],"abstract":"A core challenge in policy optimization in competitive Markov decision processes is the design of efficient optimization methods with desirable convergence and stability properties. To tackle this, we propose competitive policy optimization (CoPO), a novel policy gradient approach that exploits the game-theoretic nature of competitive games to derive policy updates. Motivated by the competitive gradient optimization method, we derive a bilinear approximation of the game objective. In contrast, off-the-shelf policy gradient methods utilize only linear approximations, and hence do not capture interactions among the players. We instantiate CoPO in two ways:(i) competitive policy gradient, and (ii) trust-region competitive policy optimization. We theoretically study these methods, and empirically investigate their behavior on a set of comprehensive, yet challenging, competitive games. We observe that they provide stable optimization, convergence to sophisticated strategies, and higher scores when played against baseline policy gradient methods.","url_abs":"https://arxiv.org/abs/2006.10611v1","url_pdf":"https://arxiv.org/pdf/2006.10611v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"competitive-policy-optimization","repo_url":"https://github.com/manish-pra/copg","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"competitive-policy-optimization","repo_url":"https://github.com/18jeffreyma/cgd_scratch_work","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"competitive-policy-optimization","repo_url":"https://github.com/18jeffreyma/multi_cmd","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"competitive-policy-optimization","repo_url":"https://github.com/manish-pra/trcopo","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"policy-gradient-methods","task_name":"Policy Gradient Methods"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2006.10611","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2006.10611"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/18jeffreyma/cgd_scratch_work","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/manish-pra/copg","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/manish-pra/trcopo","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/18jeffreyma/multi_cmd","reach":null}],"summary":{"unverified":5},"by_repo_kind":{"official":{"samples":3,"ran":0,"repositories":1},"listed":{"samples":1,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":1,"samples":[{"code_sha256_prefix":"e32c644998154590","entry":"conjugate_gradient","repo":"manish-pra/copg","repo_kind":"official","path":"copg_optim/utils.py","file_url":"https://github.com/manish-pra/copg/blob/HEAD/copg_optim/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e32c644998154590"}},{"code_sha256_prefix":"bc198e6e8818a5a0","entry":"conjugate_gradient_2trpo","repo":"manish-pra/trcopo","repo_kind":"listed","path":"trcopo_optim/utils.py","file_url":"https://github.com/manish-pra/trcopo/blob/HEAD/trcopo_optim/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bc198e6e8818a5a0"}},{"code_sha256_prefix":"796380d005e91b89","entry":"general_conjugate_gradient","repo":"manish-pra/copg","repo_kind":"official","path":"copg_optim/utils.py","file_url":"https://github.com/manish-pra/copg/blob/HEAD/copg_optim/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"796380d005e91b89"}},{"code_sha256_prefix":"ac1320ca772e1112","entry":"get_advantage","repo":"manish-pra/copg","repo_kind":"official","path":"copg_optim/critic_functions.py","file_url":"https://github.com/manish-pra/copg/blob/HEAD/copg_optim/critic_functions.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ac1320ca772e1112"}},{"code_sha256_prefix":"75d97d1a4dc46d3f","entry":"player_payoffs","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"75d97d1a4dc46d3f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}