{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/adversarially-trained-actor-critic-for","title":"Adversarially Trained Actor Critic for Offline Reinforcement Learning","arxiv_id":"2202.02446","date":"2022-02-05","proceeding":null,"authors":["Ching-An Cheng","Tengyang Xie","Nan Jiang","Alekh Agarwal"],"abstract":"We propose Adversarially Trained Actor Critic (ATAC), a new model-free algorithm for offline reinforcement learning (RL) under insufficient data coverage, based on the concept of relative pessimism. ATAC is designed as a two-player Stackelberg game: A policy actor competes against an adversarially trained value critic, who finds data-consistent scenarios where the actor is inferior to the data-collection behavior policy. We prove that, when the actor attains no regret in the two-player game, running ATAC produces a policy that provably 1) outperforms the behavior policy over a wide range of hyperparameters that control the degree of pessimism, and 2) competes with the best policy covered by data with appropriately chosen hyperparameters. Compared with existing works, notably our framework offers both theoretical guarantees for general function approximation and a deep RL implementation scalable to complex environments and large datasets. In the D4RL benchmark, ATAC consistently outperforms state-of-the-art offline RL algorithms on a range of continuous control tasks.","url_abs":"https://arxiv.org/abs/2202.02446v2","url_pdf":"https://arxiv.org/pdf/2202.02446v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"adversarially-trained-actor-critic-for","repo_url":"https://github.com/microsoft/atac","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"adversarially-trained-actor-critic-for","repo_url":"https://github.com/chinganc/lightatac","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"adversarially-trained-actor-critic-for","repo_url":"https://github.com/microsoft/lightatac","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"continuous-control","task_name":"Continuous Control"},{"task_slug":"d4rl","task_name":"D4RL"},{"task_slug":"offline-rl","task_name":"Offline RL"},{"task_slug":"reinforcement-learning","task_name":"Reinforcement Learning"},{"task_slug":"reinforcement-learning-1","task_name":"Reinforcement Learning (RL)"},{"task_slug":"continuous-control","task_name":"continuous-control"},{"task_slug":"reinforcement-learning-2","task_name":"reinforcement-learning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2202.02446","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2202.02446"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/chinganc/lightatac","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/microsoft/atac","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/microsoft/lightatac","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":1,"ran_honours":2},"by_repo_kind":{"official":{"samples":2,"ran":2,"repositories":1},"listed":{"samples":1,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"36c97c87a5d855c4","entry":"l2_projection","repo":"chinganc/lightatac","repo_kind":"listed","path":"lightATAC/atac.py","file_url":"https://github.com/chinganc/lightatac/blob/HEAD/lightATAC/atac.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"36c97c87a5d855c4"}},{"code_sha256_prefix":"2a76fc3a7969ba54","entry":"normalized_sum","repo":"microsoft/atac","repo_kind":"official","path":"src/atac/atac.py","file_url":"https://github.com/microsoft/atac/blob/HEAD/src/atac/atac.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2a76fc3a7969ba54"}},{"code_sha256_prefix":"6da18995d4a16b2e","entry":"weight_l2","repo":"microsoft/atac","repo_kind":"official","path":"src/atac/atac.py","file_url":"https://github.com/microsoft/atac/blob/HEAD/src/atac/atac.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6da18995d4a16b2e"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}