{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/model-based-policy-optimization-with","title":"Model-based Policy Optimization with Unsupervised Model Adaptation","arxiv_id":"2010.09546","date":"2020-10-19","proceeding":"NeurIPS 2020 12","authors":["Jian Shen","Han Zhao","Weinan Zhang","Yong Yu"],"abstract":"Model-based reinforcement learning methods learn a dynamics model with real data sampled from the environment and leverage it to generate simulated data to derive an agent. However, due to the potential distribution mismatch between simulated data and real data, this could lead to degraded performance. Despite much effort being devoted to reducing this distribution mismatch, existing methods fail to solve it explicitly. In this paper, we investigate how to bridge the gap between real and simulated data due to inaccurate model estimation for better policy optimization. To begin with, we first derive a lower bound of the expected return, which naturally inspires a bound maximization algorithm by aligning the simulated and real data distributions. To this end, we propose a novel model-based reinforcement learning framework AMPO, which introduces unsupervised model adaptation to minimize the integral probability metric (IPM) between feature distributions from real and simulated data. Instantiating our framework with Wasserstein-1 distance gives a practical model-based approach. Empirically, our approach achieves state-of-the-art performance in terms of sample efficiency on a range of continuous control benchmark tasks.","url_abs":"https://arxiv.org/abs/2010.09546v2","url_pdf":"https://arxiv.org/pdf/2010.09546v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"model-based-policy-optimization-with","repo_url":"https://github.com/RockySJ/ampo","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"continuous-control","task_name":"Continuous Control"},{"task_slug":"model-based-reinforcement-learning","task_name":"Model-based Reinforcement Learning"},{"task_slug":"reinforcement-learning","task_name":"Reinforcement Learning"},{"task_slug":"reinforcement-learning-1","task_name":"Reinforcement Learning (RL)"},{"task_slug":"continuous-control","task_name":"continuous-control"},{"task_slug":"model","task_name":"model"},{"task_slug":"reinforcement-learning-2","task_name":"reinforcement-learning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2010.09546","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.09546"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/RockySJ/ampo","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_honours":2,"ran_draft_wrong":2,"unverified":4},"by_repo_kind":{"official":{"samples":8,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"31deb9c7be58e00c","entry":"td_target","repo":"RockySJ/ampo","repo_kind":"official","path":"ampo.py","file_url":"https://github.com/RockySJ/ampo/blob/HEAD/ampo.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":2,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"31deb9c7be58e00c"}},{"code_sha256_prefix":"1bb5583dba32f555","entry":"format_samples_for_training","repo":"RockySJ/ampo","repo_kind":"official","path":"models/constructor.py","file_url":"https://github.com/RockySJ/ampo/blob/HEAD/models/constructor.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1bb5583dba32f555"}},{"code_sha256_prefix":"4eb5226056a1f70f","entry":"get_required_argument","repo":"RockySJ/ampo","repo_kind":"official","path":"models/utils.py","file_url":"https://github.com/RockySJ/ampo/blob/HEAD/models/utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4eb5226056a1f70f"}},{"code_sha256_prefix":"14a3c589a22e7e32","entry":"shuffle_rows","repo":"RockySJ/ampo","repo_kind":"official","path":"models/utils.py","file_url":"https://github.com/RockySJ/ampo/blob/HEAD/models/utils.py","link_basis":"plan_row","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"14a3c589a22e7e32"}},{"code_sha256_prefix":"70012af633b122df","entry":"compute_pairwise_distances","repo":"RockySJ/ampo","repo_kind":"official","path":"models/utils.py","file_url":"https://github.com/RockySJ/ampo/blob/HEAD/models/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"70012af633b122df"}},{"code_sha256_prefix":"1bb085aab9eafe21","entry":"dot","repo":"RockySJ/ampo","repo_kind":"official","path":"models/mmd.py","file_url":"https://github.com/RockySJ/ampo/blob/HEAD/models/mmd.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1bb085aab9eafe21"}},{"code_sha256_prefix":"2b18d20b2fa3d78a","entry":"mmd2","repo":"RockySJ/ampo","repo_kind":"official","path":"models/mmd.py","file_url":"https://github.com/RockySJ/ampo/blob/HEAD/models/mmd.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2b18d20b2fa3d78a"}},{"code_sha256_prefix":"3b455641d13093a1","entry":"sq_sum","repo":"RockySJ/ampo","repo_kind":"official","path":"models/mmd.py","file_url":"https://github.com/RockySJ/ampo/blob/HEAD/models/mmd.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3b455641d13093a1"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}