{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/replay-guided-adversarial-environment-design","title":"Replay-Guided Adversarial Environment Design","arxiv_id":"2110.02439","date":"2021-10-06","proceeding":"NeurIPS 2021 12","authors":["Minqi Jiang","Michael Dennis","Jack Parker-Holder","Jakob Foerster","Edward Grefenstette","Tim Rocktäschel"],"abstract":"Deep reinforcement learning (RL) agents may successfully generalize to new settings if trained on an appropriately diverse set of environment and task configurations. Unsupervised Environment Design (UED) is a promising self-supervised RL paradigm, wherein the free parameters of an underspecified environment are automatically adapted during training to the agent's capabilities, leading to the emergence of diverse training environments. Here, we cast Prioritized Level Replay (PLR), an empirically successful but theoretically unmotivated method that selectively samples randomly-generated training levels, as UED. We argue that by curating completely random levels, PLR, too, can generate novel and complex levels for effective training. This insight reveals a natural class of UED methods we call Dual Curriculum Design (DCD). Crucially, DCD includes both PLR and a popular UED algorithm, PAIRED, as special cases and inherits similar theoretical guarantees. This connection allows us to develop novel theory for PLR, providing a version with a robustness guarantee at Nash equilibria. Furthermore, our theory suggests a highly counterintuitive improvement to PLR: by stopping the agent from updating its policy on uncurated levels (training on less data), we can improve the convergence to Nash equilibria. Indeed, our experiments confirm that our new method, PLR$^{\\perp}$, obtains better results on a suite of out-of-distribution, zero-shot transfer tasks, in addition to demonstrating that PLR$^{\\perp}$ improves the performance of PAIRED, from which it inherited its theoretical framework.","url_abs":"https://arxiv.org/abs/2110.02439v2","url_pdf":"https://arxiv.org/pdf/2110.02439v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"replay-guided-adversarial-environment-design","repo_url":"https://github.com/facebookresearch/dcd","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"replay-guided-adversarial-environment-design","repo_url":"https://github.com/facebookresearch/minimax","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":null},{"paper_slug":"replay-guided-adversarial-environment-design","repo_url":"https://github.com/fictivename/dcd_mujoco","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"replay-guided-adversarial-environment-design","repo_url":"https://github.com/ucl-dark/skillhack","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"deep-reinforcement-learning","task_name":"Deep Reinforcement Learning"},{"task_slug":"reinforcement-learning-1","task_name":"Reinforcement Learning (RL)"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2110.02439","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.02439"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/facebookresearch/dcd","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ucl-dark/skillhack","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/fictivename/dcd_mujoco","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/minimax","reach":null}],"summary":{"ran":1,"unverified":3},"by_repo_kind":{"listed":{"samples":4,"ran":1,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":1,"samples":[{"code_sha256_prefix":"0ddca4701961c4c7","entry":"PLRBuffer","repo":"facebookresearch/minimax","repo_kind":"listed","path":"src/minimax/util/rl/plr.py","file_url":"https://github.com/facebookresearch/minimax/blob/HEAD/src/minimax/util/rl/plr.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0ddca4701961c4c7"}},{"code_sha256_prefix":"92c4db04a096b3db","entry":"LevelSampler","repo":"fictivename/dcd_mujoco","repo_kind":"listed","path":"level_replay/level_sampler.py","file_url":"https://github.com/fictivename/dcd_mujoco/blob/HEAD/level_replay/level_sampler.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"92c4db04a096b3db"}},{"code_sha256_prefix":"da77c74235c8a34e","entry":"PLRManager","repo":"facebookresearch/minimax","repo_kind":"listed","path":"src/minimax/util/rl/plr.py","file_url":"https://github.com/facebookresearch/minimax/blob/HEAD/src/minimax/util/rl/plr.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"da77c74235c8a34e"}},{"code_sha256_prefix":"0646e27a0ec531fa","entry":"UEDScore","repo":"facebookresearch/minimax","repo_kind":"listed","path":"src/minimax/util/rl/plr.py","file_url":"https://github.com/facebookresearch/minimax/blob/HEAD/src/minimax/util/rl/plr.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0646e27a0ec531fa"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}