{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/opendataval-a-unified-benchmark-for-data-1","title":"OpenDataVal: a Unified Benchmark for Data Valuation","arxiv_id":"2306.10577","date":"2023-06-18","proceeding":"NeurIPS 2023 11","authors":["Kevin Fu Jiang","Weixin Liang","James Zou","Yongchan Kwon"],"abstract":"Assessing the quality and impact of individual data points is critical for improving model performance and mitigating undesirable biases within the training dataset. Several data valuation algorithms have been proposed to quantify data quality, however, there lacks a systemic and standardized benchmarking system for data valuation. In this paper, we introduce OpenDataVal, an easy-to-use and unified benchmark framework that empowers researchers and practitioners to apply and compare various data valuation algorithms. OpenDataVal provides an integrated environment that includes (i) a diverse collection of image, natural language, and tabular datasets, (ii) implementations of eleven different state-of-the-art data valuation algorithms, and (iii) a prediction model API that can import any models in scikit-learn. Furthermore, we propose four downstream machine learning tasks for evaluating the quality of data values. We perform benchmarking analysis using OpenDataVal, quantifying and comparing the efficacy of state-of-the-art data valuation approaches. We find that no single algorithm performs uniformly best across all tasks, and an appropriate algorithm should be employed for a user's downstream task. OpenDataVal is publicly available at https://opendataval.github.io with comprehensive documentation. Furthermore, we provide a leaderboard where researchers can evaluate the effectiveness of their own data valuation algorithms.","url_abs":"https://arxiv.org/abs/2306.10577v3","url_pdf":"https://arxiv.org/pdf/2306.10577v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"opendataval-a-unified-benchmark-for-data-1","repo_url":"https://github.com/opendataval/opendataval","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"opendataval-a-unified-benchmark-for-data-1","repo_url":"https://github.com/caihuaiguang/CHG-Shapley-for-Data-Valuation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"benchmarking","task_name":"Benchmarking"},{"task_slug":"data-valuation","task_name":"Data Valuation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2306.10577","atlas_url":"https://app.syntology.ai/?focus=2306.10577","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.10577"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/opendataval/opendataval","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/caihuaiguang/CHG-Shapley-for-Data-Valuation","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":6,"unverified":1},"by_repo_kind":{"official":{"samples":7,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"09b21e96f9f1145c","entry":"accuracy","repo":"opendataval/opendataval","repo_kind":"official","path":"opendataval/metrics.py","file_url":"https://github.com/opendataval/opendataval/blob/HEAD/opendataval/metrics.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"09b21e96f9f1145c"}},{"code_sha256_prefix":"e4f775a9a6a730ec","entry":"get_name","repo":"opendataval/opendataval","repo_kind":"official","path":"opendataval/util.py","file_url":"https://github.com/opendataval/opendataval/blob/HEAD/opendataval/util.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e4f775a9a6a730ec"}},{"code_sha256_prefix":"ac153ea032d1ea3a","entry":"load_mediator_output","repo":"opendataval/opendataval","repo_kind":"official","path":"opendataval/util.py","file_url":"https://github.com/opendataval/opendataval/blob/HEAD/opendataval/util.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ac153ea032d1ea3a"}},{"code_sha256_prefix":"0e01cada769eb741","entry":"neg_l2","repo":"opendataval/opendataval","repo_kind":"official","path":"opendataval/metrics.py","file_url":"https://github.com/opendataval/opendataval/blob/HEAD/opendataval/metrics.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0e01cada769eb741"}},{"code_sha256_prefix":"d496216f795f6458","entry":"neg_mse","repo":"opendataval/opendataval","repo_kind":"official","path":"opendataval/metrics.py","file_url":"https://github.com/opendataval/opendataval/blob/HEAD/opendataval/metrics.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d496216f795f6458"}},{"code_sha256_prefix":"504db6072fc44357","entry":"set_random_state","repo":"opendataval/opendataval","repo_kind":"official","path":"opendataval/util.py","file_url":"https://github.com/opendataval/opendataval/blob/HEAD/opendataval/util.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"504db6072fc44357"}},{"code_sha256_prefix":"5266d8e9025c49d4","entry":"to_numpy","repo":"opendataval/opendataval","repo_kind":"official","path":"opendataval/model/api.py","file_url":"https://github.com/opendataval/opendataval/blob/HEAD/opendataval/model/api.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5266d8e9025c49d4"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}