{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/off-dynamics-reinforcement-learning-training","title":"Off-Dynamics Reinforcement Learning: Training for Transfer with Domain Classifiers","arxiv_id":"2006.13916","date":"2020-06-24","proceeding":"ICLR 2021 1","authors":["Benjamin Eysenbach","Swapnil Asawa","Shreyas Chaudhari","Sergey Levine","Ruslan Salakhutdinov"],"abstract":"We propose a simple, practical, and intuitive approach for domain adaptation in reinforcement learning. Our approach stems from the idea that the agent's experience in the source domain should look similar to its experience in the target domain. Building off of a probabilistic view of RL, we formally show that we can achieve this goal by compensating for the difference in dynamics by modifying the reward function. This modified reward function is simple to estimate by learning auxiliary classifiers that distinguish source-domain transitions from target-domain transitions. Intuitively, the modified reward function penalizes the agent for visiting states and taking actions in the source domain which are not possible in the target domain. Said another way, the agent is penalized for transitions that would indicate that the agent is interacting with the source domain, rather than the target domain. Our approach is applicable to domains with continuous states and actions and does not require learning an explicit model of the dynamics. On discrete and continuous control tasks, we illustrate the mechanics of our approach and demonstrate its scalability to high-dimensional tasks.","url_abs":"https://arxiv.org/abs/2006.13916v2","url_pdf":"https://arxiv.org/pdf/2006.13916v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"off-dynamics-reinforcement-learning-training","repo_url":"https://github.com/pauldaoudi/food","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"off-dynamics-reinforcement-learning-training","repo_url":"https://github.com/shreyasc-13/off_domain_rlkit","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"continuous-control","task_name":"Continuous Control"},{"task_slug":"domain-adaptation","task_name":"Domain Adaptation"},{"task_slug":"reinforcement-learning","task_name":"Reinforcement Learning"},{"task_slug":"reinforcement-learning-1","task_name":"Reinforcement Learning (RL)"},{"task_slug":"continuous-control","task_name":"continuous-control"},{"task_slug":"reinforcement-learning-2","task_name":"reinforcement-learning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2006.13916","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2006.13916"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/google-research/google-research","reach":null},{"provenance":"deterministic:regex_extraction","url":"https://github.com/kchua/handful-of-trials","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/JannerM/mbpo","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/shreyasc-13/off_domain_rlkit","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/pauldaoudi/food","reach":null}],"summary":{"ran_honours":3,"ran":9,"ran_draft_wrong":6,"unverified":5},"by_repo_kind":{"listed":{"samples":17,"ran":15,"repositories":2},"found_in_text":{"samples":6,"ran":3,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"31deb9c7be58e00c","entry":"td_target","repo":"JannerM/mbpo","repo_kind":"found_in_text","path":"mbpo/algorithms/mbpo.py","file_url":"https://github.com/JannerM/mbpo/blob/HEAD/mbpo/algorithms/mbpo.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":2,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"31deb9c7be58e00c"}},{"code_sha256_prefix":"85dcfc68165685af","entry":"Discriminator","repo":"pauldaoudi/food","repo_kind":"listed","path":"agents/algos_utils/darc_utils/networks_manager.py","file_url":"https://github.com/pauldaoudi/food/blob/HEAD/agents/algos_utils/darc_utils/networks_manager.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"85dcfc68165685af"}},{"code_sha256_prefix":"3072f035b0c85c12","entry":"ExpertDataset","repo":"pauldaoudi/food","repo_kind":"listed","path":"agents/algos_utils/darc_utils/networks_manager.py","file_url":"https://github.com/pauldaoudi/food/blob/HEAD/agents/algos_utils/darc_utils/networks_manager.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3072f035b0c85c12"}},{"code_sha256_prefix":"d5c0698f7ffe2457","entry":"RolloutStorage","repo":"pauldaoudi/food","repo_kind":"listed","path":"agents/algos_utils/darc_utils/networks_manager.py","file_url":"https://github.com/pauldaoudi/food/blob/HEAD/agents/algos_utils/darc_utils/networks_manager.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d5c0698f7ffe2457"}},{"code_sha256_prefix":"54a2a67d3023c95b","entry":"RunningMeanStd","repo":"pauldaoudi/food","repo_kind":"listed","path":"agents/algos_utils/darc_utils/networks_manager.py","file_url":"https://github.com/pauldaoudi/food/blob/HEAD/agents/algos_utils/darc_utils/networks_manager.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"54a2a67d3023c95b"}},{"code_sha256_prefix":"0207b2d8ce743999","entry":"_flatten_helper","repo":"pauldaoudi/food","repo_kind":"listed","path":"agents/algos_utils/darc_utils/networks_manager.py","file_url":"https://github.com/pauldaoudi/food/blob/HEAD/agents/algos_utils/darc_utils/networks_manager.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0207b2d8ce743999"}},{"code_sha256_prefix":"93d28ca67303e04b","entry":"clip_magnitude","repo":"shreyasc-13/off_domain_rlkit","repo_kind":"listed","path":"rlkit/pythonplusplus.py","file_url":"https://github.com/shreyasc-13/off_domain_rlkit/blob/HEAD/rlkit/pythonplusplus.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"93d28ca67303e04b"}},{"code_sha256_prefix":"fadf5f762ff04cd6","entry":"create_stats_ordered_dict","repo":"shreyasc-13/off_domain_rlkit","repo_kind":"listed","path":"rlkit/core/eval_util.py","file_url":"https://github.com/shreyasc-13/off_domain_rlkit/blob/HEAD/rlkit/core/eval_util.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fadf5f762ff04cd6"}},{"code_sha256_prefix":"6e7428e76d042cba","entry":"expert_data_loader","repo":"pauldaoudi/food","repo_kind":"listed","path":"agents/algos_utils/darc_utils/networks_manager.py","file_url":"https://github.com/pauldaoudi/food/blob/HEAD/agents/algos_utils/darc_utils/networks_manager.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6e7428e76d042cba"}},{"code_sha256_prefix":"1bb5583dba32f555","entry":"format_samples_for_training","repo":"JannerM/mbpo","repo_kind":"found_in_text","path":"mbpo/models/constructor.py","file_url":"https://github.com/JannerM/mbpo/blob/HEAD/mbpo/models/constructor.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1bb5583dba32f555"}},{"code_sha256_prefix":"a5d10a0b704c8db7","entry":"get_average_returns","repo":"shreyasc-13/off_domain_rlkit","repo_kind":"listed","path":"rlkit/core/eval_util.py","file_url":"https://github.com/shreyasc-13/off_domain_rlkit/blob/HEAD/rlkit/core/eval_util.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a5d10a0b704c8db7"}},{"code_sha256_prefix":"c49250e1563ad2cd","entry":"get_dataset","repo":"pauldaoudi/food","repo_kind":"listed","path":"agents/algos_utils/darc_utils/networks_manager.py","file_url":"https://github.com/pauldaoudi/food/blob/HEAD/agents/algos_utils/darc_utils/networks_manager.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c49250e1563ad2cd"}},{"code_sha256_prefix":"4eb5226056a1f70f","entry":"get_required_argument","repo":"JannerM/mbpo","repo_kind":"found_in_text","path":"mbpo/models/utils.py","file_url":"https://github.com/JannerM/mbpo/blob/HEAD/mbpo/models/utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4eb5226056a1f70f"}},{"code_sha256_prefix":"9910e2fc297f8665","entry":"identity","repo":"shreyasc-13/off_domain_rlkit","repo_kind":"listed","path":"rlkit/pythonplusplus.py","file_url":"https://github.com/shreyasc-13/off_domain_rlkit/blob/HEAD/rlkit/pythonplusplus.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9910e2fc297f8665"}},{"code_sha256_prefix":"4138766e854db009","entry":"is_numeric","repo":"shreyasc-13/off_domain_rlkit","repo_kind":"listed","path":"rlkit/pythonplusplus.py","file_url":"https://github.com/shreyasc-13/off_domain_rlkit/blob/HEAD/rlkit/pythonplusplus.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4138766e854db009"}},{"code_sha256_prefix":"0642a6a4bea7b65a","entry":"obs_batch_normalize","repo":"pauldaoudi/food","repo_kind":"listed","path":"agents/algos_utils/darc_utils/networks_manager.py","file_url":"https://github.com/pauldaoudi/food/blob/HEAD/agents/algos_utils/darc_utils/networks_manager.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0642a6a4bea7b65a"}},{"code_sha256_prefix":"5cb0c55f58279f59","entry":"simple_separated_format","repo":"shreyasc-13/off_domain_rlkit","repo_kind":"listed","path":"rlkit/core/tabulate.py","file_url":"https://github.com/shreyasc-13/off_domain_rlkit/blob/HEAD/rlkit/core/tabulate.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5cb0c55f58279f59"}},{"code_sha256_prefix":"e2024afc1a6905ff","entry":"update_mean_var_count_from_moments","repo":"pauldaoudi/food","repo_kind":"listed","path":"agents/algos_utils/darc_utils/networks_manager.py","file_url":"https://github.com/pauldaoudi/food/blob/HEAD/agents/algos_utils/darc_utils/networks_manager.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e2024afc1a6905ff"}},{"code_sha256_prefix":"4bf00ffb22a14dca","entry":"ConcatDiscriminator","repo":"pauldaoudi/food","repo_kind":"listed","path":"agents/algos_utils/darc_utils/networks_manager.py","file_url":"https://github.com/pauldaoudi/food/blob/HEAD/agents/algos_utils/darc_utils/networks_manager.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4bf00ffb22a14dca"}},{"code_sha256_prefix":"a1e7649f2401900f","entry":"NetworksManager","repo":"pauldaoudi/food","repo_kind":"listed","path":"agents/algos_utils/darc_utils/networks_manager.py","file_url":"https://github.com/pauldaoudi/food/blob/HEAD/agents/algos_utils/darc_utils/networks_manager.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a1e7649f2401900f"}},{"code_sha256_prefix":"738c96c362e9ea1d","entry":"create_read_only","repo":"kchua/handful-of-trials","repo_kind":"found_in_text","path":"dmbrl/config/default.py","file_url":"https://github.com/kchua/handful-of-trials/blob/HEAD/dmbrl/config/default.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"738c96c362e9ea1d"}},{"code_sha256_prefix":"22eb5e3f463c48d7","entry":"make_bool","repo":"kchua/handful-of-trials","repo_kind":"found_in_text","path":"dmbrl/config/default.py","file_url":"https://github.com/kchua/handful-of-trials/blob/HEAD/dmbrl/config/default.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"22eb5e3f463c48d7"}},{"code_sha256_prefix":"46bd205700c42f20","entry":"mass_center","repo":"JannerM/mbpo","repo_kind":"found_in_text","path":"mbpo/env/humanoid.py","file_url":"https://github.com/JannerM/mbpo/blob/HEAD/mbpo/env/humanoid.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"46bd205700c42f20"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}