{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/gradient-informed-proximal-policy-1","title":"Gradient Informed Proximal Policy Optimization","arxiv_id":"2312.08710","date":"2023-12-14","proceeding":"NeurIPS 2023 11","authors":["Sanghyun Son","Laura Yu Zheng","Ryan Sullivan","Yi-Ling Qiao","Ming C. Lin"],"abstract":"We introduce a novel policy learning method that integrates analytical gradients from differentiable environments with the Proximal Policy Optimization (PPO) algorithm. To incorporate analytical gradients into the PPO framework, we introduce the concept of an {\\alpha}-policy that stands as a locally superior policy. By adaptively modifying the {\\alpha} value, we can effectively manage the influence of analytical policy gradients during learning. To this end, we suggest metrics for assessing the variance and bias of analytical gradients, reducing dependence on these gradients when high variance or bias is detected. Our proposed approach outperforms baseline algorithms in various scenarios, such as function optimization, physics simulations, and traffic control environments. Our code can be found online: https://github.com/SonSang/gippo.","url_abs":"https://arxiv.org/abs/2312.08710v1","url_pdf":"https://arxiv.org/pdf/2312.08710v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"gradient-informed-proximal-policy-1","repo_url":"https://github.com/sonsang/gippo","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[],"methods":[{"method_slug":"entropy-regularization","method_name":"Entropy Regularization"},{"method_slug":"ppo","method_name":"PPO"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2312.08710","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.08710"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/SonSang/gippo","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sonsang/gippo","reach":{"status":"ok"}}],"summary":{"unverified":5},"by_repo_kind":{"official":{"samples":5,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":5,"samples":[{"code_sha256_prefix":"afd17448f52609b0","entry":"create_vecenv","repo":"sonsang/gippo","repo_kind":"official","path":"src/gippo/vecenv.py","file_url":"https://github.com/sonsang/gippo/blob/HEAD/src/gippo/vecenv.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"afd17448f52609b0"}},{"code_sha256_prefix":"d4aa6aec4a5dbfd2","entry":"get_activation_func","repo":"sonsang/gippo","repo_kind":"official","path":"src/gippo/network.py","file_url":"https://github.com/sonsang/gippo/blob/HEAD/src/gippo/network.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d4aa6aec4a5dbfd2"}},{"code_sha256_prefix":"d0cd749d322eccb6","entry":"init","repo":"sonsang/gippo","repo_kind":"official","path":"src/gippo/network.py","file_url":"https://github.com/sonsang/gippo/blob/HEAD/src/gippo/network.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d0cd749d322eccb6"}},{"code_sha256_prefix":"77184ba6b3b6f11f","entry":"seeding","repo":"sonsang/gippo","repo_kind":"official","path":"src/gippo/utils.py","file_url":"https://github.com/sonsang/gippo/blob/HEAD/src/gippo/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"77184ba6b3b6f11f"}},{"code_sha256_prefix":"5b14e5d26e807616","entry":"swap_and_flatten01","repo":"sonsang/gippo","repo_kind":"official","path":"src/gippo/utils.py","file_url":"https://github.com/sonsang/gippo/blob/HEAD/src/gippo/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5b14e5d26e807616"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}