{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/adaptive-exploration-for-data-efficient","title":"Adaptive Exploration for Data-Efficient General Value Function Evaluations","arxiv_id":"2405.07838","date":"2024-05-13","proceeding":null,"authors":["Arushi Jain","Josiah P. Hanna","Doina Precup"],"abstract":"General Value Functions (GVFs) (Sutton et al., 2011) represent predictive knowledge in reinforcement learning. Each GVF computes the expected return for a given policy, based on a unique reward. Existing methods relying on fixed behavior policies or pre-collected data often face data efficiency issues when learning multiple GVFs in parallel using off-policy methods. To address this, we introduce GVFExplorer, which adaptively learns a single behavior policy that efficiently collects data for evaluating multiple GVFs in parallel. Our method optimizes the behavior policy by minimizing the total variance in return across GVFs, thereby reducing the required environmental interactions. We use an existing temporal-difference-style variance estimator to approximate the return variance. We prove that each behavior policy update decreases the overall mean squared error in GVF predictions. We empirically show our method's performance in tabular and nonlinear function approximation settings, including Mujoco environments, with stationary and non-stationary reward signals, optimizing data usage and reducing prediction errors across multiple GVFs.","url_abs":"https://arxiv.org/abs/2405.07838v2","url_pdf":"https://arxiv.org/pdf/2405.07838v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"adaptive-exploration-for-data-efficient","repo_url":"https://github.com/arushijain94/explorationofgvfs","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"mujoco","task_name":"MuJoCo"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2405.07838","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.07838"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/arushijain94/explorationofgvfs","reach":{"status":"ok"}}],"summary":{"ran":9,"unverified":6},"by_repo_kind":{"official":{"samples":12,"ran":9,"repositories":1},"community":{"samples":3,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":15,"samples":[{"code_sha256_prefix":"bc53aa823264e72c","entry":"add_value_function","repo":"arushijain94/explorationofgvfs","repo_kind":"official","path":"VarianceReduction/src/common_td_changing_lr.py","file_url":"https://github.com/arushijain94/explorationofgvfs/blob/HEAD/VarianceReduction/src/common_td_changing_lr.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bc53aa823264e72c"}},{"code_sha256_prefix":"e1a25f9bebb55954","entry":"clip_normalize_policy","repo":"arushijain94/explorationofgvfs","repo_kind":"official","path":"VarianceReduction/src/common_bpi_changing_lr.py","file_url":"https://github.com/arushijain94/explorationofgvfs/blob/HEAD/VarianceReduction/src/common_bpi_changing_lr.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e1a25f9bebb55954"}},{"code_sha256_prefix":"73e934dec583cd6e","entry":"clip_normalize_policy","repo":"arushijain94/explorationofgvfs","repo_kind":"official","path":"VarianceReduction/src/common_martha_chnaging_lr.py","file_url":"https://github.com/arushijain94/explorationofgvfs/blob/HEAD/VarianceReduction/src/common_martha_chnaging_lr.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"73e934dec583cd6e"}},{"code_sha256_prefix":"e0ca4c397990d704","entry":"compute_mean_non_wall_states","repo":"arushijain94/explorationofgvfs","repo_kind":"official","path":"VarianceReduction/src/common_bpi_changing_lr.py","file_url":"https://github.com/arushijain94/explorationofgvfs/blob/HEAD/VarianceReduction/src/common_bpi_changing_lr.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e0ca4c397990d704"}},{"code_sha256_prefix":"c8fe5562f925ef26","entry":"create_multiple_reward","repo":"arushijain94/explorationofgvfs","repo_kind":"official","path":"VarianceReduction/src/common.py","file_url":"https://github.com/arushijain94/explorationofgvfs/blob/HEAD/VarianceReduction/src/common.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c8fe5562f925ef26"}},{"code_sha256_prefix":"8fbb675568033e2c","entry":"get_mult_simple_target_policy","repo":"arushijain94/explorationofgvfs","repo_kind":"official","path":"VarianceReduction/src/common.py","file_url":"https://github.com/arushijain94/explorationofgvfs/blob/HEAD/VarianceReduction/src/common.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8fbb675568033e2c"}},{"code_sha256_prefix":"2d87b6140723c333","entry":"get_target_policy","repo":"arushijain94/explorationofgvfs","repo_kind":"official","path":"VarianceReduction/src/common.py","file_url":"https://github.com/arushijain94/explorationofgvfs/blob/HEAD/VarianceReduction/src/common.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2d87b6140723c333"}},{"code_sha256_prefix":"4b2cfd10c65cda90","entry":"get_true_vals","repo":"arushijain94/explorationofgvfs","repo_kind":"official","path":"VarianceReduction/src/common_bpi_changing_lr.py","file_url":"https://github.com/arushijain94/explorationofgvfs/blob/HEAD/VarianceReduction/src/common_bpi_changing_lr.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4b2cfd10c65cda90"}},{"code_sha256_prefix":"ee32b502d828299f","entry":"get_var_s","repo":"arushijain94/explorationofgvfs","repo_kind":"official","path":"VarianceReduction/src/bpg.py","file_url":"https://github.com/arushijain94/explorationofgvfs/blob/HEAD/VarianceReduction/src/bpg.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ee32b502d828299f"}},{"code_sha256_prefix":"79b9dd785d50a15e","entry":"choose_action","repo":"arushijain94/gvfexplorer","repo_kind":"community","path":"VarianceReduction/src/BPI.py","file_url":"https://github.com/arushijain94/gvfexplorer/blob/HEAD/VarianceReduction/src/BPI.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"79b9dd785d50a15e"}},{"code_sha256_prefix":"74d545a970aaee61","entry":"choose_action","repo":"arushijain94/explorationofgvfs","repo_kind":"official","path":"VarianceReduction/src/bpg.py","file_url":"https://github.com/arushijain94/explorationofgvfs/blob/HEAD/VarianceReduction/src/bpg.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"74d545a970aaee61"}},{"code_sha256_prefix":"7ac31a8126242273","entry":"estimate_val_var_function","repo":"arushijain94/gvfexplorer","repo_kind":"community","path":"VarianceReduction/src/BPI.py","file_url":"https://github.com/arushijain94/gvfexplorer/blob/HEAD/VarianceReduction/src/BPI.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7ac31a8126242273"}},{"code_sha256_prefix":"55207ae764ce7593","entry":"estimate_value_function","repo":"arushijain94/gvfexplorer","repo_kind":"community","path":"VarianceReduction/src/BPI.py","file_url":"https://github.com/arushijain94/gvfexplorer/blob/HEAD/VarianceReduction/src/BPI.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"55207ae764ce7593"}},{"code_sha256_prefix":"a63c34ebc15aca18","entry":"get_csv_logger","repo":"arushijain94/explorationofgvfs","repo_kind":"official","path":"VarianceReduction/src/common_td_changing_lr.py","file_url":"https://github.com/arushijain94/explorationofgvfs/blob/HEAD/VarianceReduction/src/common_td_changing_lr.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a63c34ebc15aca18"}},{"code_sha256_prefix":"cab0e2947e5ac4c2","entry":"update_behavior_policy","repo":"arushijain94/explorationofgvfs","repo_kind":"official","path":"VarianceReduction/src/bpg.py","file_url":"https://github.com/arushijain94/explorationofgvfs/blob/HEAD/VarianceReduction/src/bpg.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"cab0e2947e5ac4c2"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}