{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/learning-guidance-rewards-with-trajectory","title":"Learning Guidance Rewards with Trajectory-space Smoothing","arxiv_id":"2010.12718","date":"2020-10-23","proceeding":"NeurIPS 2020 12","authors":["Tanmay Gangwani","Yuan Zhou","Jian Peng"],"abstract":"Long-term temporal credit assignment is an important challenge in deep reinforcement learning (RL). It refers to the ability of the agent to attribute actions to consequences that may occur after a long time interval. Existing policy-gradient and Q-learning algorithms typically rely on dense environmental rewards that provide rich short-term supervision and help with credit assignment. However, they struggle to solve tasks with delays between an action and the corresponding rewarding feedback. To make credit assignment easier, recent works have proposed algorithms to learn dense \"guidance\" rewards that could be used in place of the sparse or delayed environmental rewards. This paper is in the same vein -- starting with a surrogate RL objective that involves smoothing in the trajectory-space, we arrive at a new algorithm for learning guidance rewards. We show that the guidance rewards have an intuitive interpretation, and can be obtained without training any additional neural networks. Due to the ease of integration, we use the guidance rewards in a few popular algorithms (Q-learning, Actor-Critic, Distributional-RL) and present results in single-agent and multi-agent tasks that elucidate the benefit of our approach when the environmental rewards are sparse or delayed.","url_abs":"https://arxiv.org/abs/2010.12718v1","url_pdf":"https://arxiv.org/pdf/2010.12718v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"learning-guidance-rewards-with-trajectory","repo_url":"https://github.com/tgangwani/GuidanceRewards","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"learning-guidance-rewards-with-trajectory","repo_url":"https://github.com/baicenxiao/shaping-advice","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":null}],"tasks":[{"task_slug":"attribute","task_name":"Attribute"},{"task_slug":"deep-reinforcement-learning","task_name":"Deep Reinforcement Learning"},{"task_slug":"q-learning","task_name":"Q-Learning"},{"task_slug":"reinforcement-learning-1","task_name":"Reinforcement Learning (RL)"}],"methods":[{"method_slug":"q-learning","method_name":"Q-Learning"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2010.12718","atlas_url":"https://app.syntology.ai/?focus=2010.12718","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.12718"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/tgangwani/GuidanceRewards","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/baicenxiao/shaping-advice","reach":null}],"summary":{"ran_fixture":4,"ran_violates":1},"by_repo_kind":{"listed":{"samples":5,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":5,"samples":[{"code_sha256_prefix":"845912fb62faffa5","entry":"Check_IN","repo":"baicenxiao/shaping-advice","repo_kind":"listed","path":"maddpg/trainer/maddpg_spread.py","file_url":"https://github.com/baicenxiao/shaping-advice/blob/HEAD/maddpg/trainer/maddpg_spread.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"845912fb62faffa5"}},{"code_sha256_prefix":"f1f0bbcd3b14929e","entry":"In_P","repo":"baicenxiao/shaping-advice","repo_kind":"listed","path":"maddpg/trainer/maddpg_spread.py","file_url":"https://github.com/baicenxiao/shaping-advice/blob/HEAD/maddpg/trainer/maddpg_spread.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"f1f0bbcd3b14929e"}},{"code_sha256_prefix":"82a62f9681ee2957","entry":"SAM_NonUniform_b","repo":"baicenxiao/shaping-advice","repo_kind":"listed","path":"maddpg/trainer/maddpg_spread.py","file_url":"https://github.com/baicenxiao/shaping-advice/blob/HEAD/maddpg/trainer/maddpg_spread.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"82a62f9681ee2957"}},{"code_sha256_prefix":"73da5c8faa128c71","entry":"SAM_NonUniform_potential","repo":"baicenxiao/shaping-advice","repo_kind":"listed","path":"maddpg/trainer/maddpg_spread.py","file_url":"https://github.com/baicenxiao/shaping-advice/blob/HEAD/maddpg/trainer/maddpg_spread.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"73da5c8faa128c71"}},{"code_sha256_prefix":"e9096a66724b29e3","entry":"a_2_pa","repo":"baicenxiao/shaping-advice","repo_kind":"listed","path":"maddpg/trainer/maddpg_spread.py","file_url":"https://github.com/baicenxiao/shaping-advice/blob/HEAD/maddpg/trainer/maddpg_spread.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"e9096a66724b29e3"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}