{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/identifiability-and-generalizability-in","title":"Identifiability and Generalizability in Constrained Inverse Reinforcement Learning","arxiv_id":"2306.00629","date":"2023-06-01","proceeding":null,"authors":["Andreas Schlaginhaufen","Maryam Kamgarpour"],"abstract":"Two main challenges in Reinforcement Learning (RL) are designing appropriate reward functions and ensuring the safety of the learned policy. To address these challenges, we present a theoretical framework for Inverse Reinforcement Learning (IRL) in constrained Markov decision processes. From a convex-analytic perspective, we extend prior results on reward identifiability and generalizability to both the constrained setting and a more general class of regularizations. In particular, we show that identifiability up to potential shaping (Cao et al., 2021) is a consequence of entropy regularization and may generally no longer hold for other regularizations or in the presence of safety constraints. We also show that to ensure generalizability to new transition laws and constraints, the true reward must be identified up to a constant. Additionally, we derive a finite sample guarantee for the suboptimality of the learned rewards, and validate our results in a gridworld environment.","url_abs":"https://arxiv.org/abs/2306.00629v1","url_pdf":"https://arxiv.org/pdf/2306.00629v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"identifiability-and-generalizability-in","repo_url":"https://github.com/andrschl/cirl","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"reinforcement-learning","task_name":"Reinforcement Learning"},{"task_slug":"reinforcement-learning-1","task_name":"Reinforcement Learning (RL)"},{"task_slug":"reinforcement-learning-2","task_name":"reinforcement-learning"}],"methods":[{"method_slug":"entropy-regularization","method_name":"Entropy Regularization"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2306.00629","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.00629"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/andrschl/cirl","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":13},"by_repo_kind":{"official":{"samples":13,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"244f37859a9fb194","entry":"NPG_update","repo":"andrschl/cirl","repo_kind":"official","path":"algs/cmdp.py","file_url":"https://github.com/andrschl/cirl/blob/HEAD/algs/cmdp.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"244f37859a9fb194"}},{"code_sha256_prefix":"53c17c6603d9d565","entry":"animate_gridworld","repo":"andrschl/cirl","repo_kind":"official","path":"visualization/gridworld_vis.py","file_url":"https://github.com/andrschl/cirl/blob/HEAD/visualization/gridworld_vis.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"53c17c6603d9d565"}},{"code_sha256_prefix":"39359208e0f99183","entry":"cmdp_gda","repo":"andrschl/cirl","repo_kind":"official","path":"algs/cmdp.py","file_url":"https://github.com/andrschl/cirl/blob/HEAD/algs/cmdp.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"39359208e0f99183"}},{"code_sha256_prefix":"78fec7268a705b50","entry":"dist_to_const_line","repo":"andrschl/cirl","repo_kind":"official","path":"algs/cirl.py","file_url":"https://github.com/andrschl/cirl/blob/HEAD/algs/cirl.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"78fec7268a705b50"}},{"code_sha256_prefix":"f41acfeea124b649","entry":"empirical_expert_feature_expectation","repo":"andrschl/cirl","repo_kind":"official","path":"algs/cirl.py","file_url":"https://github.com/andrschl/cirl/blob/HEAD/algs/cirl.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f41acfeea124b649"}},{"code_sha256_prefix":"4e0247ae28bbd615","entry":"euclidean_proj_l1ball","repo":"andrschl/cirl","repo_kind":"official","path":"algs/l1_projection.py","file_url":"https://github.com/andrschl/cirl/blob/HEAD/algs/l1_projection.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4e0247ae28bbd615"}},{"code_sha256_prefix":"d13aff1a000a4d49","entry":"euclidean_proj_simplex","repo":"andrschl/cirl","repo_kind":"official","path":"algs/l1_projection.py","file_url":"https://github.com/andrschl/cirl/blob/HEAD/algs/l1_projection.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d13aff1a000a4d49"}},{"code_sha256_prefix":"c126fc1d8eb97b0c","entry":"intersection","repo":"andrschl/cirl","repo_kind":"official","path":"utils/geometric_tools.py","file_url":"https://github.com/andrschl/cirl/blob/HEAD/utils/geometric_tools.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c126fc1d8eb97b0c"}},{"code_sha256_prefix":"b19942947f52d1fe","entry":"orthogonal_complement","repo":"andrschl/cirl","repo_kind":"official","path":"utils/geometric_tools.py","file_url":"https://github.com/andrschl/cirl/blob/HEAD/utils/geometric_tools.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b19942947f52d1fe"}},{"code_sha256_prefix":"80eed4dd96b2c24f","entry":"plot_gridworld","repo":"andrschl/cirl","repo_kind":"official","path":"visualization/gridworld_vis.py","file_url":"https://github.com/andrschl/cirl/blob/HEAD/visualization/gridworld_vis.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"80eed4dd96b2c24f"}},{"code_sha256_prefix":"c6f978bd33eadfed","entry":"reward","repo":"andrschl/cirl","repo_kind":"official","path":"algs/cirl.py","file_url":"https://github.com/andrschl/cirl/blob/HEAD/algs/cirl.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c6f978bd33eadfed"}},{"code_sha256_prefix":"1dda2e3a464c543a","entry":"unit_vector","repo":"andrschl/cirl","repo_kind":"official","path":"utils/geometric_tools.py","file_url":"https://github.com/andrschl/cirl/blob/HEAD/utils/geometric_tools.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1dda2e3a464c543a"}},{"code_sha256_prefix":"2778d6a851c355ee","entry":"xi_update","repo":"andrschl/cirl","repo_kind":"official","path":"algs/cmdp.py","file_url":"https://github.com/andrschl/cirl/blob/HEAD/algs/cmdp.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2778d6a851c355ee"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}