{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2606-10228","title":"SHAPO: Sharpness-Aware Policy Optimization for Safe Exploration","arxiv_id":"2606.10228","date":"2026-06-08","proceeding":null,"authors":["Kaustubh Mani","Yann Pequignot","Vincent Mai","Liam Paull"],"abstract":"Safe exploration is a prerequisite for deploying reinforcement learning (RL) agents in safety-critical domains. In this paper, we approach safe exploration through the lens of epistemic uncertainty, where the actor's sensitivity to parameter perturbations serves as a practical proxy for regions of high uncertainty. We propose Sharpness-Aware Policy Optimization (SHAPO), a sharpness-aware policy update rule that evaluates gradients at perturbed parameters, making policy updates pessimistic with respect to the actor's epistemic uncertainty. Analytically we show that this adjustment implicitly reweighs policy gradients, amplifying the influence of rare unsafe actions while tempering contributions from already safe ones, thereby biasing learning toward conservative behavior in under-explored regions. Across several continuous-control tasks, our method consistently improves both safety and task performance over existing baselines, significantly expanding their Pareto frontiers.","url_abs":"https://arxiv.org/abs/2606.10228","url_pdf":"https://arxiv.org/pdf/2606.10228","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2606.10228","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2606.10228"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/montrealrobotics/shapo","reach":null}],"summary":{"ran":4,"ran_draft_wrong":1,"ran_honours":2,"unverified":6},"by_repo_kind":{"found_in_text":{"samples":13,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"e8e31987c5d58926","entry":"Actor","repo":"montrealrobotics/shapo","repo_kind":"found_in_text","path":"safepo/single_agent/shapo.py","file_url":"https://github.com/montrealrobotics/shapo/blob/HEAD/safepo/single_agent/shapo.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e8e31987c5d58926"}},{"code_sha256_prefix":"7af25f4312ec1e6c","entry":"VCritic","repo":"montrealrobotics/shapo","repo_kind":"found_in_text","path":"safepo/single_agent/shapo.py","file_url":"https://github.com/montrealrobotics/shapo/blob/HEAD/safepo/single_agent/shapo.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7af25f4312ec1e6c"}},{"code_sha256_prefix":"0b4c507ca061e879","entry":"build_mlp_network","repo":"montrealrobotics/shapo","repo_kind":"found_in_text","path":"safepo/single_agent/shapo.py","file_url":"https://github.com/montrealrobotics/shapo/blob/HEAD/safepo/single_agent/shapo.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0b4c507ca061e879"}},{"code_sha256_prefix":"41bcbe47eb145d13","entry":"compute_kl_constrained_perturbation","repo":"montrealrobotics/shapo","repo_kind":"found_in_text","path":"safepo/single_agent/shapo.py","file_url":"https://github.com/montrealrobotics/shapo/blob/HEAD/safepo/single_agent/shapo.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"41bcbe47eb145d13"}},{"code_sha256_prefix":"85e92e4bdd0a48f7","entry":"get_flat_gradients_from","repo":"montrealrobotics/shapo","repo_kind":"found_in_text","path":"safepo/single_agent/shapo.py","file_url":"https://github.com/montrealrobotics/shapo/blob/HEAD/safepo/single_agent/shapo.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"85e92e4bdd0a48f7"}},{"code_sha256_prefix":"c4317ff2ec911b0b","entry":"get_flat_params_from","repo":"montrealrobotics/shapo","repo_kind":"found_in_text","path":"safepo/single_agent/shapo.py","file_url":"https://github.com/montrealrobotics/shapo/blob/HEAD/safepo/single_agent/shapo.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c4317ff2ec911b0b"}},{"code_sha256_prefix":"042de5c327d75c75","entry":"log_gradient_statistics","repo":"montrealrobotics/shapo","repo_kind":"found_in_text","path":"safepo/single_agent/shapo.py","file_url":"https://github.com/montrealrobotics/shapo/blob/HEAD/safepo/single_agent/shapo.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"042de5c327d75c75"}},{"code_sha256_prefix":"0ca07951d425f2e4","entry":"ActorVCritic","repo":"montrealrobotics/shapo","repo_kind":"found_in_text","path":"safepo/single_agent/shapo.py","file_url":"https://github.com/montrealrobotics/shapo/blob/HEAD/safepo/single_agent/shapo.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0ca07951d425f2e4"}},{"code_sha256_prefix":"952f8639298e4857","entry":"compute_natural_gradient_direction","repo":"montrealrobotics/shapo","repo_kind":"found_in_text","path":"safepo/single_agent/shapo.py","file_url":"https://github.com/montrealrobotics/shapo/blob/HEAD/safepo/single_agent/shapo.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"952f8639298e4857"}},{"code_sha256_prefix":"6c2cbc030bbea5a9","entry":"compute_shapo_gradients_actor","repo":"montrealrobotics/shapo","repo_kind":"found_in_text","path":"safepo/single_agent/shapo.py","file_url":"https://github.com/montrealrobotics/shapo/blob/HEAD/safepo/single_agent/shapo.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6c2cbc030bbea5a9"}},{"code_sha256_prefix":"66ed7b1d9ea7acbc","entry":"conjugate_gradients","repo":"montrealrobotics/shapo","repo_kind":"found_in_text","path":"safepo/single_agent/shapo.py","file_url":"https://github.com/montrealrobotics/shapo/blob/HEAD/safepo/single_agent/shapo.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"66ed7b1d9ea7acbc"}},{"code_sha256_prefix":"979727939a5c3e51","entry":"fvp","repo":"montrealrobotics/shapo","repo_kind":"found_in_text","path":"safepo/single_agent/shapo.py","file_url":"https://github.com/montrealrobotics/shapo/blob/HEAD/safepo/single_agent/shapo.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"979727939a5c3e51"}},{"code_sha256_prefix":"000d35f42103806e","entry":"set_param_values_to_model","repo":"montrealrobotics/shapo","repo_kind":"found_in_text","path":"safepo/single_agent/shapo.py","file_url":"https://github.com/montrealrobotics/shapo/blob/HEAD/safepo/single_agent/shapo.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"000d35f42103806e"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.LG","source":"arxiv_2026.jsonl"},"syntology_extracted_results":null}