{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2508-06214","title":"Reparameterization Proximal Policy Optimization","arxiv_id":"2508.06214","date":"2025-08-08","proceeding":"ICML","authors":["Hai Zhong","Xun Wang","Zhuoran Li","Longbo Huang"],"abstract":"By leveraging differentiable dynamics, Reparameterization Policy Gradient (RPG) achieves high sample efficiency. However, current approaches are hindered by two critical limitations: the under-utilization of computationally expensive dynamics Jacobians and inherent training instability. While sample reuse offers a remedy for under-utilization, no prior principled framework exists, and naive attempts risk exacerbating instability. To address these challenges, we propose Reparameterization Proximal Policy Optimization (RPO). We first establish that under sample reuse, RPG naturally optimizes a PPO-style surrogate objective via Backpropagation Through Time, providing a unified framework for both on- and off-policy updates. To further ensure stability, RPO integrates a clipped policy gradient mechanism tailored for RPG and employs explicit Kullback-Leibler divergence regularization. Experimental results demonstrate that RPO maintains superior sample efficiency and consistently outperforms or achieves state-of-the-art performance across diverse tasks.","url_abs":"https://arxiv.org/abs/2508.06214","url_pdf":"https://arxiv.org/pdf/2508.06214","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2508.06214","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2508.06214"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/SonSang/gippo","reach":{"status":"ok"}},{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/etaoxing/mineral","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":9},"by_repo_kind":{"found_in_text":{"samples":9,"ran":0,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":5,"samples":[{"code_sha256_prefix":"b7cc4a6ef06ad7c4","entry":"Act","repo":"etaoxing/mineral","repo_kind":"found_in_text","path":"mineral/nets/mlp.py","file_url":"https://github.com/etaoxing/mineral/blob/HEAD/mineral/nets/mlp.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b7cc4a6ef06ad7c4"}},{"code_sha256_prefix":"28f2a00675369999","entry":"Norm","repo":"etaoxing/mineral","repo_kind":"found_in_text","path":"mineral/nets/mlp.py","file_url":"https://github.com/etaoxing/mineral/blob/HEAD/mineral/nets/mlp.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"28f2a00675369999"}},{"code_sha256_prefix":"ef86be1061ef4e01","entry":"compute_nstep_return","repo":"etaoxing/mineral","repo_kind":"found_in_text","path":"mineral/buffers/nstep_replay.py","file_url":"https://github.com/etaoxing/mineral/blob/HEAD/mineral/buffers/nstep_replay.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ef86be1061ef4e01"}},{"code_sha256_prefix":"2446e8ed75885ae8","entry":"create_buffer","repo":"etaoxing/mineral","repo_kind":"found_in_text","path":"mineral/buffers/simple_replay.py","file_url":"https://github.com/etaoxing/mineral/blob/HEAD/mineral/buffers/simple_replay.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2446e8ed75885ae8"}},{"code_sha256_prefix":"afd17448f52609b0","entry":"create_vecenv","repo":"SonSang/gippo","repo_kind":"found_in_text","path":"src/gippo/vecenv.py","file_url":"https://github.com/SonSang/gippo/blob/HEAD/src/gippo/vecenv.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"afd17448f52609b0"}},{"code_sha256_prefix":"d4aa6aec4a5dbfd2","entry":"get_activation_func","repo":"SonSang/gippo","repo_kind":"found_in_text","path":"src/gippo/network.py","file_url":"https://github.com/SonSang/gippo/blob/HEAD/src/gippo/network.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d4aa6aec4a5dbfd2"}},{"code_sha256_prefix":"d0cd749d322eccb6","entry":"init","repo":"SonSang/gippo","repo_kind":"found_in_text","path":"src/gippo/network.py","file_url":"https://github.com/SonSang/gippo/blob/HEAD/src/gippo/network.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d0cd749d322eccb6"}},{"code_sha256_prefix":"77184ba6b3b6f11f","entry":"seeding","repo":"SonSang/gippo","repo_kind":"found_in_text","path":"src/gippo/utils.py","file_url":"https://github.com/SonSang/gippo/blob/HEAD/src/gippo/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"77184ba6b3b6f11f"}},{"code_sha256_prefix":"5b14e5d26e807616","entry":"swap_and_flatten01","repo":"SonSang/gippo","repo_kind":"found_in_text","path":"src/gippo/utils.py","file_url":"https://github.com/SonSang/gippo/blob/HEAD/src/gippo/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5b14e5d26e807616"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.LG","source":"arxiv_api"},"syntology_extracted_results":null}