{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/extrapolating-beyond-suboptimal","title":"Extrapolating Beyond Suboptimal Demonstrations via Inverse Reinforcement Learning from Observations","arxiv_id":"1904.06387","date":"2019-04-12","proceeding":null,"authors":["Daniel S. Brown","Wonjoon Goo","Prabhat Nagarajan","Scott Niekum"],"abstract":"A critical flaw of existing inverse reinforcement learning (IRL) methods is their inability to significantly outperform the demonstrator. This is because IRL typically seeks a reward function that makes the demonstrator appear near-optimal, rather than inferring the underlying intentions of the demonstrator that may have been poorly executed in practice. In this paper, we introduce a novel reward-learning-from-observation algorithm, Trajectory-ranked Reward EXtrapolation (T-REX), that extrapolates beyond a set of (approximately) ranked demonstrations in order to infer high-quality reward functions from a set of potentially poor demonstrations. When combined with deep reinforcement learning, T-REX outperforms state-of-the-art imitation learning and IRL methods on multiple Atari and MuJoCo benchmark tasks and achieves performance that is often more than twice the performance of the best demonstration. We also demonstrate that T-REX is robust to ranking noise and can accurately extrapolate intention by simply watching a learner noisily improve at a task over time.","url_abs":"https://arxiv.org/abs/1904.06387v5","url_pdf":"https://arxiv.org/pdf/1904.06387v5.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"extrapolating-beyond-suboptimal","repo_url":"https://github.com/hiwonjoon/ICML2019-TREX","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"tf","reach":null},{"paper_slug":"extrapolating-beyond-suboptimal","repo_url":"https://github.com/francidellungo/Minigrid_HCI-project","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"extrapolating-beyond-suboptimal","repo_url":"https://github.com/opendilab/DI-engine","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"deep-reinforcement-learning","task_name":"Deep Reinforcement Learning"},{"task_slug":"imitation-learning","task_name":"Imitation Learning"},{"task_slug":"mujoco","task_name":"MuJoCo"},{"task_slug":"reinforcement-learning","task_name":"Reinforcement Learning"},{"task_slug":"reinforcement-learning-1","task_name":"Reinforcement Learning (RL)"},{"task_slug":"reinforcement-learning-2","task_name":"reinforcement-learning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1904.06387","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1904.06387"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/francidellungo/Minigrid_HCI-project","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/hiwonjoon/ICML2019-TREX","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/opendilab/DI-engine","reach":null}],"summary":{"ran_draft_wrong":2,"unverified":1},"by_repo_kind":{"official":{"samples":3,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"e300d11a27b3ebfa","entry":"create_training_data","repo":"hiwonjoon/ICML2019-TREX","repo_kind":"official","path":"atari/LearnAtariReward.py","file_url":"https://github.com/hiwonjoon/ICML2019-TREX/blob/HEAD/atari/LearnAtariReward.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e300d11a27b3ebfa"}},{"code_sha256_prefix":"d47ac370b45f99a2","entry":"create_training_data_from_mturk","repo":"hiwonjoon/ICML2019-TREX","repo_kind":"official","path":"atari/LearnAtariMTurkRankings.py","file_url":"https://github.com/hiwonjoon/ICML2019-TREX/blob/HEAD/atari/LearnAtariMTurkRankings.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d47ac370b45f99a2"}},{"code_sha256_prefix":"e5de5dc1ff3645df","entry":"calc_accuracy","repo":"hiwonjoon/ICML2019-TREX","repo_kind":"official","path":"atari/LearnAtariMTurkRankings.py","file_url":"https://github.com/hiwonjoon/ICML2019-TREX/blob/HEAD/atari/LearnAtariMTurkRankings.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e5de5dc1ff3645df"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}