{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/pipeline-psro-a-scalable-approach-for-finding","title":"Pipeline PSRO: A Scalable Approach for Finding Approximate Nash Equilibria in Large Games","arxiv_id":"2006.08555","date":"2020-06-15","proceeding":"NeurIPS 2020 12","authors":["Stephen McAleer","John Lanier","Roy Fox","Pierre Baldi"],"abstract":"Finding approximate Nash equilibria in zero-sum imperfect-information games is challenging when the number of information states is large. Policy Space Response Oracles (PSRO) is a deep reinforcement learning algorithm grounded in game theory that is guaranteed to converge to an approximate Nash equilibrium. However, PSRO requires training a reinforcement learning policy at each iteration, making it too slow for large games. We show through counterexamples and experiments that DCH and Rectified PSRO, two existing approaches to scaling up PSRO, fail to converge even in small games. We introduce Pipeline PSRO (P2SRO), the first scalable general method for finding approximate Nash equilibria in large zero-sum imperfect-information games. P2SRO is able to parallelize PSRO with convergence guarantees by maintaining a hierarchical pipeline of reinforcement learning workers, each training against the policies generated by lower levels in the hierarchy. We show that unlike existing methods, P2SRO converges to an approximate Nash equilibrium, and does so faster as the number of parallel workers increases, across a variety of imperfect information games. We also introduce an open-source environment for Barrage Stratego, a variant of Stratego with an approximate game tree complexity of $10^{50}$. P2SRO is able to achieve state-of-the-art performance on Barrage Stratego and beats all existing bots. Experiment code is available athttps://github.com/JBLanier/pipeline-psro.","url_abs":"https://arxiv.org/abs/2006.08555v2","url_pdf":"https://arxiv.org/pdf/2006.08555v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"pipeline-psro-a-scalable-approach-for-finding","repo_url":"https://github.com/JBLanier/pipeline-psro","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"pipeline-psro-a-scalable-approach-for-finding","repo_url":"https://github.com/JBLanier/distributed-rl-for-imperfect-info-games","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"deep-reinforcement-learning","task_name":"Deep Reinforcement Learning"},{"task_slug":"reinforcement-learning","task_name":"Reinforcement Learning"},{"task_slug":"reinforcement-learning-1","task_name":"Reinforcement Learning (RL)"},{"task_slug":"reinforcement-learning-2","task_name":"reinforcement-learning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2006.08555","atlas_url":"https://app.syntology.ai/?focus=2006.08555","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2006.08555"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/JBLanier/pipeline-psro","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/JBLanier/distributed-rl-for-imperfect-info-games","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":7},"by_repo_kind":{"official":{"samples":7,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"52754f30e667f367","entry":"create_reference_policy_update_callback_for_self_play_eval","repo":"JBLanier/pipeline-psro","repo_kind":"official","path":"multiplayer-rl/mprl/rl/common/custom_eval_callbacks.py","file_url":"https://github.com/JBLanier/pipeline-psro/blob/HEAD/multiplayer-rl/mprl/rl/common/custom_eval_callbacks.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"52754f30e667f367"}},{"code_sha256_prefix":"07bb49d778f2619a","entry":"create_worker_process_pool_with_ids","repo":"JBLanier/pipeline-psro","repo_kind":"official","path":"multiplayer-rl/mprl/utils.py","file_url":"https://github.com/JBLanier/pipeline-psro/blob/HEAD/multiplayer-rl/mprl/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"07bb49d778f2619a"}},{"code_sha256_prefix":"da0e4c55ea1a63f8","entry":"load_training_data_from_file","repo":"JBLanier/pipeline-psro","repo_kind":"official","path":"multiplayer-rl/mprl/utils.py","file_url":"https://github.com/JBLanier/pipeline-psro/blob/HEAD/multiplayer-rl/mprl/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"da0e4c55ea1a63f8"}},{"code_sha256_prefix":"49507ebdfbe4ea29","entry":"policy_with_dirichlet_noise","repo":"JBLanier/pipeline-psro","repo_kind":"official","path":"multiplayer-rl/mprl/utils.py","file_url":"https://github.com/JBLanier/pipeline-psro/blob/HEAD/multiplayer-rl/mprl/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"49507ebdfbe4ea29"}},{"code_sha256_prefix":"f48bf6417161da6f","entry":"pretty_print","repo":"JBLanier/pipeline-psro","repo_kind":"official","path":"multiplayer-rl/mprl/utility_services/utils.py","file_url":"https://github.com/JBLanier/pipeline-psro/blob/HEAD/multiplayer-rl/mprl/utility_services/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f48bf6417161da6f"}},{"code_sha256_prefix":"42371142597f2af5","entry":"seconds_to_text","repo":"JBLanier/pipeline-psro","repo_kind":"official","path":"multiplayer-rl/mprl/utility_services/utils.py","file_url":"https://github.com/JBLanier/pipeline-psro/blob/HEAD/multiplayer-rl/mprl/utility_services/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"42371142597f2af5"}},{"code_sha256_prefix":"d7263ae5e7760caa","entry":"with_base_config","repo":"JBLanier/pipeline-psro","repo_kind":"official","path":"multiplayer-rl/mprl/utility_services/utils.py","file_url":"https://github.com/JBLanier/pipeline-psro/blob/HEAD/multiplayer-rl/mprl/utility_services/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d7263ae5e7760caa"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}