{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/learning-values-across-many-orders-of","title":"Learning values across many orders of magnitude","arxiv_id":"1602.07714","date":"2016-02-24","proceeding":"NeurIPS 2016 12","authors":["Hado van Hasselt","Arthur Guez","Matteo Hessel","Volodymyr Mnih","David Silver"],"abstract":"Most learning algorithms are not invariant to the scale of the function that\nis being approximated. We propose to adaptively normalize the targets used in\nlearning. This is useful in value-based reinforcement learning, where the\nmagnitude of appropriate value approximations can change over time when we\nupdate the policy of behavior. Our main motivation is prior work on learning to\nplay Atari games, where the rewards were all clipped to a predetermined range.\nThis clipping facilitates learning across many different games with a single\nlearning algorithm, but a clipped reward function can result in qualitatively\ndifferent behavior. Using the adaptive normalization we can remove this\ndomain-specific heuristic without diminishing overall performance.","url_abs":"http://arxiv.org/abs/1602.07714v2","url_pdf":"http://arxiv.org/pdf/1602.07714v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[],"tasks":[{"task_slug":"atari-games","task_name":"Atari Games"},{"task_slug":"reinforcement-learning","task_name":"Reinforcement Learning"},{"task_slug":"reinforcement-learning-1","task_name":"Reinforcement Learning (RL)"},{"task_slug":"reinforcement-learning-2","task_name":"reinforcement-learning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/atari-games-on-atari-2600-alien","task":"Atari Games","dataset":"Atari 2600 Alien","model":"DDQN+Pop-Art noop","rank_in_archive_order":25,"of":49,"metrics":{"Score":"3213.5"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-amidar","task":"Atari Games","dataset":"Atari 2600 Amidar","model":"DDQN+Pop-Art noop","rank_in_archive_order":27,"of":48,"metrics":{"Score":"782.5"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-assault","task":"Atari Games","dataset":"Atari 2600 Assault","model":"DDQN+Pop-Art noop","rank_in_archive_order":19,"of":45,"metrics":{"Score":"9011.6"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-asterix","task":"Atari Games","dataset":"Atari 2600 Asterix","model":"DDQN+Pop-Art noop","rank_in_archive_order":29,"of":49,"metrics":{"Score":"18919.5"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-asteroids","task":"Atari Games","dataset":"Atari 2600 Asteroids","model":"DDQN+Pop-Art noop","rank_in_archive_order":21,"of":43,"metrics":{"Score":"2869.3"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-atlantis","task":"Atari Games","dataset":"Atari 2600 Atlantis","model":"DDQN+Pop-Art noop","rank_in_archive_order":31,"of":42,"metrics":{"Score":"340076.0"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-bank-heist","task":"Atari Games","dataset":"Atari 2600 Bank Heist","model":"DDQN+Pop-Art noop","rank_in_archive_order":22,"of":45,"metrics":{"Score":"1103.3"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-battle-zone","task":"Atari Games","dataset":"Atari 2600 Battle Zone","model":"DDQN+Pop-Art noop","rank_in_archive_order":45,"of":47,"metrics":{"Score":"8220.0"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-beam-rider","task":"Atari Games","dataset":"Atari 2600 Beam Rider","model":"DDQN+Pop-Art noop","rank_in_archive_order":35,"of":49,"metrics":{"Score":"8299.4"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-berzerk","task":"Atari Games","dataset":"Atari 2600 Berzerk","model":"DDQN+Pop-Art noop","rank_in_archive_order":25,"of":39,"metrics":{"Score":"1199.6"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-bowling","task":"Atari Games","dataset":"Atari 2600 Bowling","model":"DDQN+Pop-Art noop","rank_in_archive_order":12,"of":44,"metrics":{"Score":"102.1"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-boxing","task":"Atari Games","dataset":"Atari 2600 Boxing","model":"DDQN+Pop-Art noop","rank_in_archive_order":18,"of":45,"metrics":{"Score":"99.3"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-breakout","task":"Atari Games","dataset":"Atari 2600 Breakout","model":"DDQN+Pop-Art noop","rank_in_archive_order":38,"of":58,"metrics":{"Score":"344.1"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-centipede","task":"Atari Games","dataset":"Atari 2600 Centipede","model":"DDQN+Pop-Art noop","rank_in_archive_order":12,"of":45,"metrics":{"Score":"49065.8"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-chopper-command","task":"Atari Games","dataset":"Atari 2600 Chopper Command","model":"DDQN+Pop-Art noop","rank_in_archive_order":44,"of":45,"metrics":{"Score":"775.0"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-crazy-climber","task":"Atari Games","dataset":"Atari 2600 Crazy Climber","model":"DDQN+Pop-Art noop","rank_in_archive_order":31,"of":49,"metrics":{"Score":"119679.0"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-demon-attack","task":"Atari Games","dataset":"Atari 2600 Demon Attack","model":"DDQN+Pop-Art noop","rank_in_archive_order":28,"of":46,"metrics":{"Score":"63644.9"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-double-dunk","task":"Atari Games","dataset":"Atari 2600 Double Dunk","model":"DDQN+Pop-Art noop","rank_in_archive_order":35,"of":43,"metrics":{"Score":"-11.5"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-enduro","task":"Atari Games","dataset":"Atari 2600 Enduro","model":"DDQN+Pop-Art noop","rank_in_archive_order":21,"of":48,"metrics":{"Score":"2002.1"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-fishing-derby","task":"Atari Games","dataset":"Atari 2600 Fishing Derby","model":"DDQN+Pop-Art noop","rank_in_archive_order":14,"of":44,"metrics":{"Score":"45.1"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-freeway","task":"Atari Games","dataset":"Atari 2600 Freeway","model":"DDQN+Pop-Art noop","rank_in_archive_order":16,"of":59,"metrics":{"Score":"33.4"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-frostbite","task":"Atari Games","dataset":"Atari 2600 Frostbite","model":"DDQN+Pop-Art noop","rank_in_archive_order":24,"of":53,"metrics":{"Score":"3469.6"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-gopher","task":"Atari Games","dataset":"Atari 2600 Gopher","model":"DDQN+Pop-Art noop","rank_in_archive_order":16,"of":43,"metrics":{"Score":"56218.2"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-gravitar","task":"Atari Games","dataset":"Atari 2600 Gravitar","model":"DDQN+Pop-Art noop","rank_in_archive_order":32,"of":53,"metrics":{"Score":"483.5"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-hero","task":"Atari Games","dataset":"Atari 2600 HERO","model":"DDQN+Pop-Art noop","rank_in_archive_order":36,"of":44,"metrics":{"Score":"14225.2"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-ice-hockey","task":"Atari Games","dataset":"Atari 2600 Ice Hockey","model":"DDQN+Pop-Art noop","rank_in_archive_order":29,"of":43,"metrics":{"Score":"-4.1"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-james-bond","task":"Atari Games","dataset":"Atari 2600 James Bond","model":"DDQN+Pop-Art noop","rank_in_archive_order":36,"of":45,"metrics":{"Score":"507.5"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-kangaroo","task":"Atari Games","dataset":"Atari 2600 Kangaroo","model":"DDQN+Pop-Art noop","rank_in_archive_order":16,"of":47,"metrics":{"Score":"13150.0"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-krull","task":"Atari Games","dataset":"Atari 2600 Krull","model":"DDQN+Pop-Art noop","rank_in_archive_order":19,"of":45,"metrics":{"Score":"9745.1"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-kung-fu-master","task":"Atari Games","dataset":"Atari 2600 Kung-Fu Master","model":"DDQN+Pop-Art noop","rank_in_archive_order":27,"of":43,"metrics":{"Score":"34393.0"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-ms-pacman","task":"Atari Games","dataset":"Atari 2600 Ms. Pacman","model":"DDQN+Pop-Art noop","rank_in_archive_order":20,"of":47,"metrics":{"Score":"4963.8"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-name-this-game","task":"Atari Games","dataset":"Atari 2600 Name This Game","model":"DDQN+Pop-Art noop","rank_in_archive_order":14,"of":43,"metrics":{"Score":"15851.2"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-pong","task":"Atari Games","dataset":"Atari 2600 Pong","model":"DDQN+Pop-Art noop","rank_in_archive_order":25,"of":52,"metrics":{"Score":"20.6"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-private-eye","task":"Atari Games","dataset":"Atari 2600 Private Eye","model":"DDQN+Pop-Art noop","rank_in_archive_order":30,"of":52,"metrics":{"Score":"286.7"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-qbert","task":"Atari Games","dataset":"Atari 2600 Q*Bert","model":"DDQN+Pop-Art noop","rank_in_archive_order":43,"of":57,"metrics":{"Score":"5236.8"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-river-raid","task":"Atari Games","dataset":"Atari 2600 River Raid","model":"DDQN+Pop-Art noop","rank_in_archive_order":24,"of":41,"metrics":{"Score":"12530.8"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-road-runner","task":"Atari Games","dataset":"Atari 2600 Road Runner","model":"DDQN+Pop-Art noop","rank_in_archive_order":27,"of":44,"metrics":{"Score":"47770.0"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-robotank","task":"Atari Games","dataset":"Atari 2600 Robotank","model":"DDQN+Pop-Art noop","rank_in_archive_order":17,"of":42,"metrics":{"Score":"64.3"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-seaquest","task":"Atari Games","dataset":"Atari 2600 Seaquest","model":"DDQN+Pop-Art noop","rank_in_archive_order":21,"of":57,"metrics":{"Score":"10932.3"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-space-invaders","task":"Atari Games","dataset":"Atari 2600 Space Invaders","model":"DDQN+Pop-Art noop","rank_in_archive_order":33,"of":55,"metrics":{"Score":"2589.7"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-star-gunner","task":"Atari Games","dataset":"Atari 2600 Star Gunner","model":"DDQN+Pop-Art noop","rank_in_archive_order":42,"of":43,"metrics":{"Score":"589.0"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-tennis","task":"Atari Games","dataset":"Atari 2600 Tennis","model":"DDQN+Pop-Art noop","rank_in_archive_order":14,"of":43,"metrics":{"Score":"12.1"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-time-pilot","task":"Atari Games","dataset":"Atari 2600 Time Pilot","model":"DDQN+Pop-Art noop","rank_in_archive_order":39,"of":44,"metrics":{"Score":"4870.0"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-tutankham","task":"Atari Games","dataset":"Atari 2600 Tutankham","model":"DDQN+Pop-Art noop","rank_in_archive_order":28,"of":44,"metrics":{"Score":"183.9"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-up-and-down","task":"Atari Games","dataset":"Atari 2600 Up and Down","model":"DDQN+Pop-Art noop","rank_in_archive_order":30,"of":44,"metrics":{"Score":"22474.4"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-venture","task":"Atari Games","dataset":"Atari 2600 Venture","model":"DDQN+Pop-Art noop","rank_in_archive_order":17,"of":55,"metrics":{"Score":"1172.0"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-video-pinball","task":"Atari Games","dataset":"Atari 2600 Video Pinball","model":"DDQN+Pop-Art noop","rank_in_archive_order":35,"of":42,"metrics":{"Score":"56287.0"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-wizard-of-wor","task":"Atari Games","dataset":"Atari 2600 Wizard of Wor","model":"DDQN+Pop-Art noop","rank_in_archive_order":40,"of":41,"metrics":{"Score":"483.0"},"uses_additional_data":false},{"leaderboard":"/sota/atari-games-on-atari-2600-zaxxon","task":"Atari Games","dataset":"Atari 2600 Zaxxon","model":"DDQN+Pop-Art noop","rank_in_archive_order":18,"of":41,"metrics":{"Score":"14402.0"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1602.07714","mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}