{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/principal-agent-reward-shaping-in-mdps","title":"Principal-Agent Reward Shaping in MDPs","arxiv_id":"2401.00298","date":"2023-12-30","proceeding":null,"authors":["Omer Ben-Porat","Yishay Mansour","Michal Moshkovitz","Boaz Taitler"],"abstract":"Principal-agent problems arise when one party acts on behalf of another, leading to conflicts of interest. The economic literature has extensively studied principal-agent problems, and recent work has extended this to more complex scenarios such as Markov Decision Processes (MDPs). In this paper, we further explore this line of research by investigating how reward shaping under budget constraints can improve the principal's utility. We study a two-player Stackelberg game where the principal and the agent have different reward functions, and the agent chooses an MDP policy for both players. The principal offers an additional reward to the agent, and the agent picks their policy selfishly to maximize their reward, which is the sum of the original and the offered reward. Our results establish the NP-hardness of the problem and offer polynomial approximation algorithms for two classes of instances: Stochastic trees and deterministic decision processes with a finite horizon.","url_abs":"https://arxiv.org/abs/2401.00298v1","url_pdf":"https://arxiv.org/pdf/2401.00298v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"principal-agent-reward-shaping-in-mdps","repo_url":"https://github.com/boaztait/principal_agent_mdp","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"none","reach":{"status":"ok"}}],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2401.00298","atlas_url":"https://app.syntology.ai/?focus=2401.00298","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.00298"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/boaztait/principal_agent_mdp","reach":{"status":"ok"}}],"summary":{"ran":8},"by_repo_kind":{"official":{"samples":8,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":8,"samples":[{"code_sha256_prefix":"761905f1fa694b49","entry":"find_opt_VP","repo":"boaztait/principal_agent_mdp","repo_kind":"official","path":"up_mdp_deterministic_solver.py","file_url":"https://github.com/boaztait/principal_agent_mdp/blob/HEAD/up_mdp_deterministic_solver.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"761905f1fa694b49"}},{"code_sha256_prefix":"4ef9f58eab61f6ab","entry":"generate_ddp","repo":"boaztait/principal_agent_mdp","repo_kind":"official","path":"up_mdp_ddp.py","file_url":"https://github.com/boaztait/principal_agent_mdp/blob/HEAD/up_mdp_ddp.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4ef9f58eab61f6ab"}},{"code_sha256_prefix":"82f77bd789dbaf00","entry":"get_VA","repo":"boaztait/principal_agent_mdp","repo_kind":"official","path":"up_mdp_ddp.py","file_url":"https://github.com/boaztait/principal_agent_mdp/blob/HEAD/up_mdp_ddp.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"82f77bd789dbaf00"}},{"code_sha256_prefix":"a39d0b8998c64336","entry":"maxVA","repo":"boaztait/principal_agent_mdp","repo_kind":"official","path":"up_mdp_ddp.py","file_url":"https://github.com/boaztait/principal_agent_mdp/blob/HEAD/up_mdp_ddp.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a39d0b8998c64336"}},{"code_sha256_prefix":"2485cb29a37b944d","entry":"maxkey","repo":"boaztait/principal_agent_mdp","repo_kind":"official","path":"up_mdp_deterministic_solver.py","file_url":"https://github.com/boaztait/principal_agent_mdp/blob/HEAD/up_mdp_deterministic_solver.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2485cb29a37b944d"}},{"code_sha256_prefix":"bd3d10133a3b9db6","entry":"rec_brute_solve","repo":"boaztait/principal_agent_mdp","repo_kind":"official","path":"brute_force_deterministic_solver.py","file_url":"https://github.com/boaztait/principal_agent_mdp/blob/HEAD/brute_force_deterministic_solver.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bd3d10133a3b9db6"}},{"code_sha256_prefix":"ca369eced14b3f9a","entry":"solve_brute_force","repo":"boaztait/principal_agent_mdp","repo_kind":"official","path":"brute_force_deterministic_solver.py","file_url":"https://github.com/boaztait/principal_agent_mdp/blob/HEAD/brute_force_deterministic_solver.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ca369eced14b3f9a"}},{"code_sha256_prefix":"2fdd1a0ddf09a7df","entry":"sortkey","repo":"boaztait/principal_agent_mdp","repo_kind":"official","path":"up_mdp_deterministic_solver.py","file_url":"https://github.com/boaztait/principal_agent_mdp/blob/HEAD/up_mdp_deterministic_solver.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2fdd1a0ddf09a7df"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}