{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/successor-uncertainties-exploration-and","title":"Successor Uncertainties: Exploration and Uncertainty in Temporal Difference Learning","arxiv_id":"1810.06530","date":"2018-10-15","proceeding":"NeurIPS 2019 12","authors":["David Janz","Jiri Hron","Przemysław Mazur","Katja Hofmann","José Miguel Hernández-Lobato","Sebastian Tschiatschek"],"abstract":"Posterior sampling for reinforcement learning (PSRL) is an effective method for balancing exploration and exploitation in reinforcement learning. Randomised value functions (RVF) can be viewed as a promising approach to scaling PSRL. However, we show that most contemporary algorithms combining RVF with neural network function approximation do not possess the properties which make PSRL effective, and provably fail in sparse reward problems. Moreover, we find that propagation of uncertainty, a property of PSRL previously thought important for exploration, does not preclude this failure. We use these insights to design Successor Uncertainties (SU), a cheap and easy to implement RVF algorithm that retains key properties of PSRL. SU is highly effective on hard tabular exploration benchmarks. Furthermore, on the Atari 2600 domain, it surpasses human performance on 38 of 49 games tested (achieving a median human normalised score of 2.09), and outperforms its closest RVF competitor, Bootstrapped DQN, on 36 of those.","url_abs":"https://arxiv.org/abs/1810.06530v5","url_pdf":"https://arxiv.org/pdf/1810.06530v5.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"successor-uncertainties-exploration-and","repo_url":"https://github.com/DavidJanz/successor_uncertainties_atari","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"successor-uncertainties-exploration-and","repo_url":"https://github.com/DavidJanz/successor_uncertainties_tabular","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"decision-making","task_name":"Decision Making"},{"task_slug":"reinforcement-learning","task_name":"Reinforcement Learning"},{"task_slug":"reinforcement-learning-1","task_name":"Reinforcement Learning (RL)"},{"task_slug":"reinforcement-learning-2","task_name":"reinforcement-learning"}],"methods":[{"method_slug":"convolution","method_name":"Convolution"},{"method_slug":"dqn","method_name":"DQN"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"q-learning","method_name":"Q-Learning"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/1810.06530","atlas_url":"https://app.syntology.ai/?focus=1810.06530","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1810.06530"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/DavidJanz/successor_uncertainties_tabular","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/DavidJanz/successor_uncertainties_atari","reach":null}],"summary":{"ran_violates":1,"ran_fixture":3,"ran_honours":1,"ran_draft_wrong":2},"by_repo_kind":{"listed":{"samples":7,"ran":7,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":7,"samples":[{"code_sha256_prefix":"a1164f2c12752323","entry":"emulator_time_exit","repo":"DavidJanz/successor_uncertainties_atari","repo_kind":"listed","path":"run_test.py","file_url":"https://github.com/DavidJanz/successor_uncertainties_atari/blob/HEAD/run_test.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a1164f2c12752323"}},{"code_sha256_prefix":"c5e92f1d08cd7598","entry":"get_color","repo":"DavidJanz/successor_uncertainties_tabular","repo_kind":"listed","path":"plotting/plot_scaling.py","file_url":"https://github.com/DavidJanz/successor_uncertainties_tabular/blob/HEAD/plotting/plot_scaling.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c5e92f1d08cd7598"}},{"code_sha256_prefix":"70df6725d99bbfb9","entry":"get_line","repo":"DavidJanz/successor_uncertainties_tabular","repo_kind":"listed","path":"plotting/plot_scaling.py","file_url":"https://github.com/DavidJanz/successor_uncertainties_tabular/blob/HEAD/plotting/plot_scaling.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"70df6725d99bbfb9"}},{"code_sha256_prefix":"42daa8f89616ba8f","entry":"play_episode","repo":"DavidJanz/successor_uncertainties_atari","repo_kind":"listed","path":"run_atari.py","file_url":"https://github.com/DavidJanz/successor_uncertainties_atari/blob/HEAD/run_atari.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"42daa8f89616ba8f"}},{"code_sha256_prefix":"6c5a23a441cc3ec1","entry":"play_test_episode","repo":"DavidJanz/successor_uncertainties_atari","repo_kind":"listed","path":"run_test.py","file_url":"https://github.com/DavidJanz/successor_uncertainties_atari/blob/HEAD/run_test.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6c5a23a441cc3ec1"}},{"code_sha256_prefix":"4352e80bae2d0815","entry":"read_pickle_scaling","repo":"DavidJanz/successor_uncertainties_tabular","repo_kind":"listed","path":"plotting/plot_scaling.py","file_url":"https://github.com/DavidJanz/successor_uncertainties_tabular/blob/HEAD/plotting/plot_scaling.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4352e80bae2d0815"}},{"code_sha256_prefix":"5fcf7685d99a18af","entry":"s2ms_str","repo":"DavidJanz/successor_uncertainties_tabular","repo_kind":"listed","path":"run/__main__.py","file_url":"https://github.com/DavidJanz/successor_uncertainties_tabular/blob/HEAD/run/__main__.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5fcf7685d99a18af"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}