{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2601-21306","title":"The Surprising Difficulty of Search in Model-Based Reinforcement Learning","arxiv_id":"2601.21306","date":"2026-01-29","proceeding":"ICML","authors":["Wei-Di Chang","Mikael Henaff","Brandon Amos","Gregory Dudek","Scott Fujimoto"],"abstract":"This paper investigates search in model-based reinforcement learning (RL). Conventional wisdom holds that long-term predictions and compounding errors are the primary obstacles for model-based RL. We challenge this view, showing that search is not a drop-in replacement for a learned policy. Surprisingly, we find that search can harm performance even when the model is highly accurate. Instead, we show that mitigating overestimation bias matters more than improving model or value function accuracy. Building on this insight, we identify that taking the minimum over an ensemble of value functions effectively addresses this bias and enables effective search, achieving state-of-the-art performance across multiple popular benchmark domains.","url_abs":"https://arxiv.org/abs/2601.21306","url_pdf":"https://arxiv.org/pdf/2601.21306","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2601.21306","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2601.21306"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/facebookresearch/MRSQ","reach":null}],"summary":{"ran":2,"ran_draft_wrong":1,"unverified":1},"by_repo_kind":{"found_in_text":{"samples":4,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":4,"samples":[{"code_sha256_prefix":"629dd4056e0a2525","entry":"BaseMLP","repo":"facebookresearch/MRSQ","repo_kind":"found_in_text","path":"MRSQ/MRSQ.py","file_url":"https://github.com/facebookresearch/MRSQ/blob/HEAD/MRSQ/MRSQ.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"629dd4056e0a2525"}},{"code_sha256_prefix":"5c4890147136a329","entry":"Value","repo":"facebookresearch/MRSQ","repo_kind":"found_in_text","path":"MRSQ/MRSQ.py","file_url":"https://github.com/facebookresearch/MRSQ/blob/HEAD/MRSQ/MRSQ.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"5c4890147136a329"}},{"code_sha256_prefix":"954768de78cae9c2","entry":"ln_activ","repo":"facebookresearch/MRSQ","repo_kind":"found_in_text","path":"MRSQ/MRSQ.py","file_url":"https://github.com/facebookresearch/MRSQ/blob/HEAD/MRSQ/MRSQ.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"954768de78cae9c2"}},{"code_sha256_prefix":"575b4a52d71bbed3","entry":"weight_init","repo":"facebookresearch/MRSQ","repo_kind":"found_in_text","path":"MRSQ/MRSQ.py","file_url":"https://github.com/facebookresearch/MRSQ/blob/HEAD/MRSQ/MRSQ.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"575b4a52d71bbed3"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.LG","source":"arxiv_2026.jsonl"},"syntology_extracted_results":null}