{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/tree-search-based-policy-optimization-under","title":"Tree Search-Based Policy Optimization under Stochastic Execution Delay","arxiv_id":"2404.05440","date":"2024-04-08","proceeding":null,"authors":["David Valensi","Esther Derman","Shie Mannor","Gal Dalal"],"abstract":"The standard formulation of Markov decision processes (MDPs) assumes that the agent's decisions are executed immediately. However, in numerous realistic applications such as robotics or healthcare, actions are performed with a delay whose value can even be stochastic. In this work, we introduce stochastic delayed execution MDPs, a new formalism addressing random delays without resorting to state augmentation. We show that given observed delay values, it is sufficient to perform a policy search in the class of Markov policies in order to reach optimal performance, thus extending the deterministic fixed delay case. Armed with this insight, we devise DEZ, a model-based algorithm that optimizes over the class of Markov policies. DEZ leverages Monte-Carlo tree search similar to its non-delayed variant EfficientZero to accurately infer future states from the action queue. Thus, it handles delayed execution while preserving the sample efficiency of EfficientZero. Through a series of experiments on the Atari suite, we demonstrate that although the previous baseline outperforms the naive method in scenarios with constant delay, it underperforms in the face of stochastic delays. In contrast, our approach significantly outperforms the baselines, for both constant and stochastic delays. The code is available at http://github.com/davidva1/Delayed-EZ .","url_abs":"https://arxiv.org/abs/2404.05440v1","url_pdf":"https://arxiv.org/pdf/2404.05440v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"tree-search-based-policy-optimization-under","repo_url":"https://github.com/davidva1/delayed-ez","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"GPL-3.0"}}],"tasks":[],"methods":[{"method_slug":"monte-carlo-tree-search","method_name":"Monte-Carlo Tree Search"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2404.05440","atlas_url":"https://app.syntology.ai/?focus=2404.05440","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.05440"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/davidva1/Delayed-EZ","reach":{"status":"ok","spdx":"GPL-3.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/davidva1/delayed-ez","reach":{"status":"ok","spdx":"GPL-3.0"}}],"summary":{"ran":6,"ran_draft_wrong":1,"ran_fixture":1},"by_repo_kind":{"official":{"samples":8,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":8,"samples":[{"code_sha256_prefix":"347b76ed10742d2a","entry":"concat_output","repo":"davidva1/Delayed-EZ","repo_kind":"official","path":"core/model.py","file_url":"https://github.com/davidva1/Delayed-EZ/blob/HEAD/core/model.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"347b76ed10742d2a"}},{"code_sha256_prefix":"05286bc5e4e73ead","entry":"concat_output_value","repo":"davidva1/Delayed-EZ","repo_kind":"official","path":"core/model.py","file_url":"https://github.com/davidva1/Delayed-EZ/blob/HEAD/core/model.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"05286bc5e4e73ead"}},{"code_sha256_prefix":"18fbe72ffac82ee3","entry":"conv3x3","repo":"davidva1/Delayed-EZ","repo_kind":"official","path":"config/atari/model.py","file_url":"https://github.com/davidva1/Delayed-EZ/blob/HEAD/config/atari/model.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"18fbe72ffac82ee3"}},{"code_sha256_prefix":"461cd9ba90efb8ac","entry":"make_atari","repo":"davidva1/Delayed-EZ","repo_kind":"official","path":"core/utils.py","file_url":"https://github.com/davidva1/Delayed-EZ/blob/HEAD/core/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"461cd9ba90efb8ac"}},{"code_sha256_prefix":"f12fcbff81a09011","entry":"make_results_dir","repo":"davidva1/Delayed-EZ","repo_kind":"official","path":"core/utils.py","file_url":"https://github.com/davidva1/Delayed-EZ/blob/HEAD/core/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"f12fcbff81a09011"}},{"code_sha256_prefix":"7af8d641a39b4ee5","entry":"mlp","repo":"davidva1/Delayed-EZ","repo_kind":"official","path":"config/atari/model.py","file_url":"https://github.com/davidva1/Delayed-EZ/blob/HEAD/config/atari/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"7af8d641a39b4ee5"}},{"code_sha256_prefix":"fa63400bb2942968","entry":"renormalize","repo":"davidva1/Delayed-EZ","repo_kind":"official","path":"core/model.py","file_url":"https://github.com/davidva1/Delayed-EZ/blob/HEAD/core/model.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"fa63400bb2942968"}},{"code_sha256_prefix":"7cf8a53360f599af","entry":"select_action","repo":"davidva1/Delayed-EZ","repo_kind":"official","path":"core/utils.py","file_url":"https://github.com/davidva1/Delayed-EZ/blob/HEAD/core/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"7cf8a53360f599af"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}