{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/future-dependent-value-based-off-policy-1","title":"Future-Dependent Value-Based Off-Policy Evaluation in POMDPs","arxiv_id":"2207.13081","date":"2022-07-26","proceeding":"NeurIPS 2023 11","authors":["Masatoshi Uehara","Haruka Kiyohara","Andrew Bennett","Victor Chernozhukov","Nan Jiang","Nathan Kallus","Chengchun Shi","Wen Sun"],"abstract":"We study off-policy evaluation (OPE) for partially observable MDPs (POMDPs) with general function approximation. Existing methods such as sequential importance sampling estimators and fitted-Q evaluation suffer from the curse of horizon in POMDPs. To circumvent this problem, we develop a novel model-free OPE method by introducing future-dependent value functions that take future proxies as inputs. Future-dependent value functions play similar roles as classical value functions in fully-observable MDPs. We derive a new Bellman equation for future-dependent value functions as conditional moment equations that use history proxies as instrumental variables. We further propose a minimax learning method to learn future-dependent value functions using the new Bellman equation. We obtain the PAC result, which implies our OPE estimator is consistent as long as futures and histories contain sufficient information about latent states, and the Bellman completeness. Finally, we extend our methods to learning of dynamics and establish the connection between our approach and the well-known spectral learning methods in POMDPs.","url_abs":"https://arxiv.org/abs/2207.13081v2","url_pdf":"https://arxiv.org/pdf/2207.13081v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"future-dependent-value-based-off-policy-1","repo_url":"https://github.com/aiueola/neurips2023-future-dependent-ope","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"off-policy-evaluation","task_name":"Off-policy evaluation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2207.13081","atlas_url":"https://app.syntology.ai/?focus=2207.13081","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2207.13081"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/aiueola/neurips2023-future-dependent-ope","reach":null}],"summary":{"ran":2,"ran_fixture":1,"unverified":5},"by_repo_kind":{"official":{"samples":8,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"02424e3e43163bbe","entry":"ContinuousStateLSTMVfunction","repo":"aiueola/neurips2023-future-dependent-ope","repo_kind":"official","path":"src/ope/value_based.py","file_url":"https://github.com/aiueola/neurips2023-future-dependent-ope/blob/HEAD/src/ope/value_based.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"02424e3e43163bbe"}},{"code_sha256_prefix":"c5356c33529b4921","entry":"DiscreteStateLSTMVfunction","repo":"aiueola/neurips2023-future-dependent-ope","repo_kind":"official","path":"src/ope/value_based.py","file_url":"https://github.com/aiueola/neurips2023-future-dependent-ope/blob/HEAD/src/ope/value_based.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c5356c33529b4921"}},{"code_sha256_prefix":"00dc3e8f76d44fbd","entry":"to_tensor","repo":"aiueola/neurips2023-future-dependent-ope","repo_kind":"official","path":"src/ope/value_based.py","file_url":"https://github.com/aiueola/neurips2023-future-dependent-ope/blob/HEAD/src/ope/value_based.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"00dc3e8f76d44fbd"}},{"code_sha256_prefix":"c8f089d208af6389","entry":"BaseNeuralValueBasedOffPolicyEstimator","repo":"aiueola/neurips2023-future-dependent-ope","repo_kind":"official","path":"src/ope/value_based.py","file_url":"https://github.com/aiueola/neurips2023-future-dependent-ope/blob/HEAD/src/ope/value_based.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c8f089d208af6389"}},{"code_sha256_prefix":"67bf91a82b09ce05","entry":"BaseOffPolicyEstimator","repo":"aiueola/neurips2023-future-dependent-ope","repo_kind":"official","path":"src/ope/value_based.py","file_url":"https://github.com/aiueola/neurips2023-future-dependent-ope/blob/HEAD/src/ope/value_based.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"67bf91a82b09ce05"}},{"code_sha256_prefix":"626f06ce600233a5","entry":"BasePolicy","repo":"aiueola/neurips2023-future-dependent-ope","repo_kind":"official","path":"src/ope/value_based.py","file_url":"https://github.com/aiueola/neurips2023-future-dependent-ope/blob/HEAD/src/ope/value_based.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"626f06ce600233a5"}},{"code_sha256_prefix":"3839b6d96f39e770","entry":"BaseValueBasedOffPolicyEstimator","repo":"aiueola/neurips2023-future-dependent-ope","repo_kind":"official","path":"src/ope/value_based.py","file_url":"https://github.com/aiueola/neurips2023-future-dependent-ope/blob/HEAD/src/ope/value_based.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3839b6d96f39e770"}},{"code_sha256_prefix":"7f505a88e43b7d23","entry":"NeuralFutureDependentValueBasedOPE","repo":"aiueola/neurips2023-future-dependent-ope","repo_kind":"official","path":"src/ope/value_based.py","file_url":"https://github.com/aiueola/neurips2023-future-dependent-ope/blob/HEAD/src/ope/value_based.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7f505a88e43b7d23"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}