{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/reinforcement-learning-from-delayed","title":"Reinforcement Learning from Delayed Observations via World Models","arxiv_id":"2403.12309","date":"2024-03-18","proceeding":null,"authors":["Armin Karamzade","KyungMin Kim","Montek Kalsi","Roy Fox"],"abstract":"In standard reinforcement learning settings, agents typically assume immediate feedback about the effects of their actions after taking them. However, in practice, this assumption may not hold true due to physical constraints and can significantly impact the performance of learning algorithms. In this paper, we address observation delays in partially observable environments. We propose leveraging world models, which have shown success in integrating past observations and learning dynamics, to handle observation delays. By reducing delayed POMDPs to delayed MDPs with world models, our methods can effectively handle partial observability, where existing approaches achieve sub-optimal performance or degrade quickly as observability decreases. Experiments suggest that one of our methods can outperform a naive model-based approach by up to 250%. Moreover, we evaluate our methods on visual delayed environments, for the first time showcasing delay-aware reinforcement learning continuous control with visual observations.","url_abs":"https://arxiv.org/abs/2403.12309v2","url_pdf":"https://arxiv.org/pdf/2403.12309v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"reinforcement-learning-from-delayed","repo_url":"https://github.com/indylab/delayeddreamer","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"jax","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"continuous-control","task_name":"Continuous Control"},{"task_slug":"reinforcement-learning","task_name":"Reinforcement Learning"},{"task_slug":"continuous-control","task_name":"continuous-control"},{"task_slug":"reinforcement-learning-2","task_name":"reinforcement-learning"}],"methods":[{"method_slug":"focus","method_name":"Focus"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2403.12309","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.12309"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/indylab/delayeddreamer","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":4,"unverified":6},"by_repo_kind":{"official":{"samples":10,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"24fe13b4f847da6b","entry":"cast_to_compute","repo":"indylab/delayeddreamer","repo_kind":"official","path":"dreamerv3/jaxutils.py","file_url":"https://github.com/indylab/delayeddreamer/blob/HEAD/dreamerv3/jaxutils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"24fe13b4f847da6b"}},{"code_sha256_prefix":"bbd25fa32893729e","entry":"pure","repo":"indylab/delayeddreamer","repo_kind":"official","path":"dreamerv3/ninjax.py","file_url":"https://github.com/indylab/delayeddreamer/blob/HEAD/dreamerv3/ninjax.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bbd25fa32893729e"}},{"code_sha256_prefix":"e5b1910ea05891b1","entry":"subsample","repo":"indylab/delayeddreamer","repo_kind":"official","path":"dreamerv3/jaxutils.py","file_url":"https://github.com/indylab/delayeddreamer/blob/HEAD/dreamerv3/jaxutils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e5b1910ea05891b1"}},{"code_sha256_prefix":"67c6d2355562342f","entry":"tensorstats","repo":"indylab/delayeddreamer","repo_kind":"official","path":"dreamerv3/jaxutils.py","file_url":"https://github.com/indylab/delayeddreamer/blob/HEAD/dreamerv3/jaxutils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"67c6d2355562342f"}},{"code_sha256_prefix":"4b830637b16db4c8","entry":"convert","repo":"indylab/delayeddreamer","repo_kind":"official","path":"dreamerv3/embodied/core/basics.py","file_url":"https://github.com/indylab/delayeddreamer/blob/HEAD/dreamerv3/embodied/core/basics.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4b830637b16db4c8"}},{"code_sha256_prefix":"3360b32511bc42d4","entry":"format_","repo":"indylab/delayeddreamer","repo_kind":"official","path":"dreamerv3/embodied/core/basics.py","file_url":"https://github.com/indylab/delayeddreamer/blob/HEAD/dreamerv3/embodied/core/basics.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3360b32511bc42d4"}},{"code_sha256_prefix":"03dd934385b4d240","entry":"get_act","repo":"indylab/delayeddreamer","repo_kind":"official","path":"dreamerv3/nets.py","file_url":"https://github.com/indylab/delayeddreamer/blob/HEAD/dreamerv3/nets.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"03dd934385b4d240"}},{"code_sha256_prefix":"1ad869363216c281","entry":"grad","repo":"indylab/delayeddreamer","repo_kind":"official","path":"dreamerv3/ninjax.py","file_url":"https://github.com/indylab/delayeddreamer/blob/HEAD/dreamerv3/ninjax.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1ad869363216c281"}},{"code_sha256_prefix":"45b97b51433849a7","entry":"rng","repo":"indylab/delayeddreamer","repo_kind":"official","path":"dreamerv3/ninjax.py","file_url":"https://github.com/indylab/delayeddreamer/blob/HEAD/dreamerv3/ninjax.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"45b97b51433849a7"}},{"code_sha256_prefix":"5ba274a4e18b46f3","entry":"treemap","repo":"indylab/delayeddreamer","repo_kind":"official","path":"dreamerv3/embodied/core/basics.py","file_url":"https://github.com/indylab/delayeddreamer/blob/HEAD/dreamerv3/embodied/core/basics.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5ba274a4e18b46f3"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}