{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/one-risk-to-rule-them-all-a-risk-sensitive-1","title":"One Risk to Rule Them All: A Risk-Sensitive Perspective on Model-Based Offline Reinforcement Learning","arxiv_id":"2212.00124","date":"2022-11-30","proceeding":"NeurIPS 2023 11","authors":["Marc Rigter","Bruno Lacerda","Nick Hawes"],"abstract":"Offline reinforcement learning (RL) is suitable for safety-critical domains where online exploration is too costly or dangerous. In such safety-critical settings, decision-making should take into consideration the risk of catastrophic outcomes. In other words, decision-making should be risk-sensitive. Previous works on risk in offline RL combine together offline RL techniques, to avoid distributional shift, with risk-sensitive RL algorithms, to achieve risk-sensitivity. In this work, we propose risk-sensitivity as a mechanism to jointly address both of these issues. Our model-based approach is risk-averse to both epistemic and aleatoric uncertainty. Risk-aversion to epistemic uncertainty prevents distributional shift, as areas not covered by the dataset have high epistemic uncertainty. Risk-aversion to aleatoric uncertainty discourages actions that may result in poor outcomes due to environment stochasticity. Our experiments show that our algorithm achieves competitive performance on deterministic benchmarks, and outperforms existing approaches for risk-sensitive objectives in stochastic domains.","url_abs":"https://arxiv.org/abs/2212.00124v3","url_pdf":"https://arxiv.org/pdf/2212.00124v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"one-risk-to-rule-them-all-a-risk-sensitive-1","repo_url":"https://github.com/marc-rigter/1r2r","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"all","task_name":"All"},{"task_slug":"decision-making","task_name":"Decision Making"},{"task_slug":"offline-rl","task_name":"Offline RL"},{"task_slug":"reinforcement-learning-1","task_name":"Reinforcement Learning (RL)"},{"task_slug":"sensitivity","task_name":"Sensitivity"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2212.00124","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2212.00124"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/tianheyu927/mopo","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/takuseno/d3rlpy","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/sfujim/TD3_BC","reach":null},{"provenance":"deterministic:regex_extraction","url":"https://github.com/marc-rigter/rambo","reach":null},{"provenance":"deterministic:regex_extraction","url":"https://github.com/ikostrikov/implicit_q_learning","reach":null},{"provenance":"deterministic:regex_extraction","url":"https://github.com/marc-rigter/1R2R","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/nuria95/O-RAAC","reach":null},{"provenance":"deterministic:regex_extraction","url":"https://github.com/JasonMa2016/CODAC","reach":null},{"provenance":"deterministic:regex_extraction","url":"https://github.com/microsoft/lightATAC","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/marc-rigter/1r2r","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_honours":2,"ran":5,"ran_fixture":1,"ran_draft_wrong":4,"unverified":11},"by_repo_kind":{"official":{"samples":8,"ran":3,"repositories":1},"found_in_text":{"samples":13,"ran":7,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":2,"samples":[{"code_sha256_prefix":"31deb9c7be58e00c","entry":"td_target","repo":"marc-rigter/1R2R","repo_kind":"official","path":"_1R2R/algorithms/_1R2R.py","file_url":"https://github.com/marc-rigter/1R2R/blob/HEAD/_1R2R/algorithms/_1R2R.py","link_basis":"plan_row","language":"python","status":"ran_honours","verification_level":2,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"31deb9c7be58e00c"}},{"code_sha256_prefix":"d354e29969a8eecf","entry":"CPW","repo":"nuria95/O-RAAC","repo_kind":"found_in_text","path":"oraaclib/agent/ORAAC.py","file_url":"https://github.com/nuria95/O-RAAC/blob/HEAD/oraaclib/agent/ORAAC.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d354e29969a8eecf"}},{"code_sha256_prefix":"0a92b095c96f1001","entry":"Observation","repo":"nuria95/O-RAAC","repo_kind":"found_in_text","path":"oraaclib/agent/ORAAC.py","file_url":"https://github.com/nuria95/O-RAAC/blob/HEAD/oraaclib/agent/ORAAC.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0a92b095c96f1001"}},{"code_sha256_prefix":"d0dc0545525ce106","entry":"Power","repo":"nuria95/O-RAAC","repo_kind":"found_in_text","path":"oraaclib/agent/ORAAC.py","file_url":"https://github.com/nuria95/O-RAAC/blob/HEAD/oraaclib/agent/ORAAC.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d0dc0545525ce106"}},{"code_sha256_prefix":"f0d06853888e809c","entry":"Wang_distortion","repo":"nuria95/O-RAAC","repo_kind":"found_in_text","path":"oraaclib/agent/ORAAC.py","file_url":"https://github.com/nuria95/O-RAAC/blob/HEAD/oraaclib/agent/ORAAC.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f0d06853888e809c"}},{"code_sha256_prefix":"0665de8d83eef80b","entry":"compute_cvar","repo":"nuria95/O-RAAC","repo_kind":"found_in_text","path":"oraaclib/agent/ORAAC.py","file_url":"https://github.com/nuria95/O-RAAC/blob/HEAD/oraaclib/agent/ORAAC.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0665de8d83eef80b"}},{"code_sha256_prefix":"1bb5583dba32f555","entry":"format_samples_for_training","repo":"marc-rigter/1R2R","repo_kind":"official","path":"_1R2R/models/constructor.py","file_url":"https://github.com/marc-rigter/1R2R/blob/HEAD/_1R2R/models/constructor.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1bb5583dba32f555"}},{"code_sha256_prefix":"4eb5226056a1f70f","entry":"get_required_argument","repo":"marc-rigter/1R2R","repo_kind":"official","path":"_1R2R/models/utils.py","file_url":"https://github.com/marc-rigter/1R2R/blob/HEAD/_1R2R/models/utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4eb5226056a1f70f"}},{"code_sha256_prefix":"36c97c87a5d855c4","entry":"l2_projection","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"36c97c87a5d855c4"}},{"code_sha256_prefix":"2a76fc3a7969ba54","entry":"normalized_sum","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"2a76fc3a7969ba54"}},{"code_sha256_prefix":"24475939b27492ce","entry":"quantile_huber_loss","repo":"nuria95/O-RAAC","repo_kind":"found_in_text","path":"oraaclib/agent/ORAAC.py","file_url":"https://github.com/nuria95/O-RAAC/blob/HEAD/oraaclib/agent/ORAAC.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"24475939b27492ce"}},{"code_sha256_prefix":"fd64627ef6698537","entry":"split","repo":"microsoft/lightATAC","repo_kind":"found_in_text","path":"lightATAC/atac.py","file_url":"https://github.com/microsoft/lightATAC/blob/HEAD/lightATAC/atac.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fd64627ef6698537"}},{"code_sha256_prefix":"385289a2f103a41b","entry":"ATAC","repo":"microsoft/lightATAC","repo_kind":"found_in_text","path":"lightATAC/atac.py","file_url":"https://github.com/microsoft/lightATAC/blob/HEAD/lightATAC/atac.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"385289a2f103a41b"}},{"code_sha256_prefix":"c5bf09ce02ff5964","entry":"AbstractAgent","repo":"nuria95/O-RAAC","repo_kind":"found_in_text","path":"oraaclib/agent/ORAAC.py","file_url":"https://github.com/nuria95/O-RAAC/blob/HEAD/oraaclib/agent/ORAAC.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c5bf09ce02ff5964"}},{"code_sha256_prefix":"7d39619c9ae882e2","entry":"ORAAC","repo":"nuria95/O-RAAC","repo_kind":"found_in_text","path":"oraaclib/agent/ORAAC.py","file_url":"https://github.com/nuria95/O-RAAC/blob/HEAD/oraaclib/agent/ORAAC.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7d39619c9ae882e2"}},{"code_sha256_prefix":"95422abceacb6468","entry":"compute_batched","repo":"microsoft/lightATAC","repo_kind":"found_in_text","path":"lightATAC/atac.py","file_url":"https://github.com/microsoft/lightATAC/blob/HEAD/lightATAC/atac.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"95422abceacb6468"}},{"code_sha256_prefix":"b69606fbe4ae1c6d","entry":"gen_dataset","repo":"marc-rigter/1R2R","repo_kind":"official","path":"_1R2R/off_policy/loader.py","file_url":"https://github.com/marc-rigter/1R2R/blob/HEAD/_1R2R/off_policy/loader.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b69606fbe4ae1c6d"}},{"code_sha256_prefix":"46bd205700c42f20","entry":"mass_center","repo":"marc-rigter/1R2R","repo_kind":"official","path":"_1R2R/env/humanoid.py","file_url":"https://github.com/marc-rigter/1R2R/blob/HEAD/_1R2R/env/humanoid.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"46bd205700c42f20"}},{"code_sha256_prefix":"721b133d3e718b81","entry":"restore_pool","repo":"marc-rigter/1R2R","repo_kind":"official","path":"_1R2R/off_policy/loader.py","file_url":"https://github.com/marc-rigter/1R2R/blob/HEAD/_1R2R/off_policy/loader.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"721b133d3e718b81"}},{"code_sha256_prefix":"4a8a13bf2658940e","entry":"restore_pool_d4rl","repo":"marc-rigter/1R2R","repo_kind":"official","path":"_1R2R/off_policy/loader.py","file_url":"https://github.com/marc-rigter/1R2R/blob/HEAD/_1R2R/off_policy/loader.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4a8a13bf2658940e"}},{"code_sha256_prefix":"d5ae5ee54ffb75d0","entry":"slice_dist","repo":"microsoft/lightATAC","repo_kind":"found_in_text","path":"lightATAC/atac.py","file_url":"https://github.com/microsoft/lightATAC/blob/HEAD/lightATAC/atac.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d5ae5ee54ffb75d0"}},{"code_sha256_prefix":"633dcc6fe7c3ecc3","entry":"spectral_norm","repo":"marc-rigter/1R2R","repo_kind":"official","path":"_1R2R/models/utils.py","file_url":"https://github.com/marc-rigter/1R2R/blob/HEAD/_1R2R/models/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"633dcc6fe7c3ecc3"}},{"code_sha256_prefix":"c9923434b7a879c9","entry":"split_dist","repo":"microsoft/lightATAC","repo_kind":"found_in_text","path":"lightATAC/atac.py","file_url":"https://github.com/microsoft/lightATAC/blob/HEAD/lightATAC/atac.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c9923434b7a879c9"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}