{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/the-effects-of-reward-misspecification-1","title":"The Effects of Reward Misspecification: Mapping and Mitigating Misaligned Models","arxiv_id":"2201.03544","date":"2022-01-10","proceeding":"ICLR 2022 4","authors":["Alexander Pan","Kush Bhatia","Jacob Steinhardt"],"abstract":"Reward hacking -- where RL agents exploit gaps in misspecified reward functions -- has been widely observed, but not yet systematically studied. To understand how reward hacking arises, we construct four RL environments with misspecified rewards. We investigate reward hacking as a function of agent capabilities: model capacity, action space resolution, observation space noise, and training time. More capable agents often exploit reward misspecifications, achieving higher proxy reward and lower true reward than less capable agents. Moreover, we find instances of phase transitions: capability thresholds at which the agent's behavior qualitatively shifts, leading to a sharp decrease in the true reward. Such phase transitions pose challenges to monitoring the safety of ML systems. To address this, we propose an anomaly detection task for aberrant policies and offer several baseline detectors.","url_abs":"https://arxiv.org/abs/2201.03544v2","url_pdf":"https://arxiv.org/pdf/2201.03544v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"the-effects-of-reward-misspecification-1","repo_url":"https://github.com/aypan17/reward-misspecification","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":null}],"tasks":[{"task_slug":"anomaly-detection","task_name":"Anomaly Detection"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2201.03544","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2201.03544"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/aypan17/reward-misspecification","reach":null}],"summary":{"ran_draft_wrong":18,"ran_honours":7,"unverified":1},"by_repo_kind":{"official":{"samples":26,"ran":25,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"c07b1065dc495f48","entry":"commute_time","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"flow/envs/reward_wrapper.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/flow/envs/reward_wrapper.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c07b1065dc495f48"}},{"code_sha256_prefix":"e9123707cba84b2c","entry":"compute_baseline_loss","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"atari/torchbeast/monobeast.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/atari/torchbeast/monobeast.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e9123707cba84b2c"}},{"code_sha256_prefix":"a27a9dc97eefda7f","entry":"compute_entropy_loss","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"atari/torchbeast/monobeast.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/atari/torchbeast/monobeast.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a27a9dc97eefda7f"}},{"code_sha256_prefix":"d6d7e0906f1c57b2","entry":"compute_policy_gradient_loss","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"atari/torchbeast/monobeast.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/atari/torchbeast/monobeast.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d6d7e0906f1c57b2"}},{"code_sha256_prefix":"552d152dfb50144b","entry":"create_client","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"flow/utils/aimsun/api.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/flow/utils/aimsun/api.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"552d152dfb50144b"}},{"code_sha256_prefix":"f134e49cf4945317","entry":"desired_velocity","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"flow/envs/reward_wrapper.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/flow/envs/reward_wrapper.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f134e49cf4945317"}},{"code_sha256_prefix":"fd4884464a9c2e1e","entry":"desired_velocity_all","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"flow/envs/reward_wrapper.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/flow/envs/reward_wrapper.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fd4884464a9c2e1e"}},{"code_sha256_prefix":"87788ea605276b52","entry":"forward_progress","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"flow/envs/reward_wrapper.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/flow/envs/reward_wrapper.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"87788ea605276b52"}},{"code_sha256_prefix":"46efffa70825570e","entry":"global_vel_all","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"flow/envs/reward_wrapper.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/flow/envs/reward_wrapper.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"46efffa70825570e"}},{"code_sha256_prefix":"82c961f3c9edac5d","entry":"local_desired_vel_all","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"flow/envs/reward_wrapper.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/flow/envs/reward_wrapper.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"82c961f3c9edac5d"}},{"code_sha256_prefix":"3d5f714f72b94405","entry":"local_desired_vel_first","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"flow/envs/reward_wrapper.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/flow/envs/reward_wrapper.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3d5f714f72b94405"}},{"code_sha256_prefix":"2d66a065fa9320a0","entry":"local_desired_vel_last","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"flow/envs/reward_wrapper.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/flow/envs/reward_wrapper.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2d66a065fa9320a0"}},{"code_sha256_prefix":"3488cca1a136f258","entry":"local_desired_velocity","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"flow/envs/reward_wrapper.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/flow/envs/reward_wrapper.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3488cca1a136f258"}},{"code_sha256_prefix":"9ba1bd737249ca19","entry":"min_delay_unscaled","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"flow/envs/reward_wrapper.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/flow/envs/reward_wrapper.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9ba1bd737249ca19"}},{"code_sha256_prefix":"f5d4db1d8aff7042","entry":"outflow","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"flow/envs/reward_wrapper.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/flow/envs/reward_wrapper.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f5d4db1d8aff7042"}},{"code_sha256_prefix":"073d130eaee08fc2","entry":"penalize_accel","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"flow/envs/reward_wrapper.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/flow/envs/reward_wrapper.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"073d130eaee08fc2"}},{"code_sha256_prefix":"b382253230203a5d","entry":"penalize_boolean_lane_change","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"flow/envs/reward_wrapper.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/flow/envs/reward_wrapper.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b382253230203a5d"}},{"code_sha256_prefix":"a410281ad545b4eb","entry":"penalize_cars","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"flow/envs/reward_wrapper.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/flow/envs/reward_wrapper.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a410281ad545b4eb"}},{"code_sha256_prefix":"ec79dbe1ac601d22","entry":"penalize_headway","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"flow/envs/reward_wrapper.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/flow/envs/reward_wrapper.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ec79dbe1ac601d22"}},{"code_sha256_prefix":"51025c4dc296185e","entry":"penalize_lane_change","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"flow/envs/reward_wrapper.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/flow/envs/reward_wrapper.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"51025c4dc296185e"}},{"code_sha256_prefix":"b1caf703f297303e","entry":"penalize_light_change","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"flow/envs/reward_wrapper.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/flow/envs/reward_wrapper.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b1caf703f297303e"}},{"code_sha256_prefix":"74ee9f91bbf19e6e","entry":"penalize_mean_delay","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"flow/envs/reward_wrapper.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/flow/envs/reward_wrapper.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"74ee9f91bbf19e6e"}},{"code_sha256_prefix":"0cc033a1bf65b8fa","entry":"penalize_standstill","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"flow/envs/reward_wrapper.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/flow/envs/reward_wrapper.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0cc033a1bf65b8fa"}},{"code_sha256_prefix":"c3fa7c4e60c383aa","entry":"penalize_still","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"flow/envs/reward_wrapper.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/flow/envs/reward_wrapper.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c3fa7c4e60c383aa"}},{"code_sha256_prefix":"c05e384bb14b1ea0","entry":"upweight_bus_vel","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"flow/envs/reward_wrapper.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/flow/envs/reward_wrapper.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c05e384bb14b1ea0"}},{"code_sha256_prefix":"2750bb69aec9993d","entry":"ProxyRewardEnv","repo":"aypan17/reward-misspecification","repo_kind":"official","path":"flow/envs/reward_wrapper.py","file_url":"https://github.com/aypan17/reward-misspecification/blob/HEAD/flow/envs/reward_wrapper.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2750bb69aec9993d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}