{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/quantile-risk-control-a-flexible-framework","title":"Quantile Risk Control: A Flexible Framework for Bounding the Probability of High-Loss Predictions","arxiv_id":"2212.13629","date":"2022-12-27","proceeding":null,"authors":["Jake C. Snell","Thomas P. Zollo","Zhun Deng","Toniann Pitassi","Richard Zemel"],"abstract":"Rigorous guarantees about the performance of predictive algorithms are necessary in order to ensure their responsible use. Previous work has largely focused on bounding the expected loss of a predictor, but this is not sufficient in many risk-sensitive applications where the distribution of errors is important. In this work, we propose a flexible framework to produce a family of bounds on quantiles of the loss distribution incurred by a predictor. Our method takes advantage of the order statistics of the observed loss values rather than relying on the sample mean alone. We show that a quantile is an informative way of quantifying predictive performance, and that our framework applies to a variety of quantile-based metrics, each targeting important subsets of the data distribution. We analyze the theoretical properties of our proposed method and demonstrate its ability to rigorously control loss quantiles on several real-world datasets.","url_abs":"https://arxiv.org/abs/2212.13629v1","url_pdf":"https://arxiv.org/pdf/2212.13629v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"quantile-risk-control-a-flexible-framework","repo_url":"https://github.com/jakesnell/quantile-risk-control","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2212.13629","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2212.13629"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jakesnell/quantile-risk-control","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":15},"by_repo_kind":{"official":{"samples":15,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"0b988a3c5dbab48a","entry":"WSR_mu_plus","repo":"jakesnell/quantile-risk-control","repo_kind":"official","path":"var_control/methods.py","file_url":"https://github.com/jakesnell/quantile-risk-control/blob/HEAD/var_control/methods.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0b988a3c5dbab48a"}},{"code_sha256_prefix":"0eeb198c33d03831","entry":"balanced_accuracy","repo":"jakesnell/quantile-risk-control","repo_kind":"official","path":"var_control/metric.py","file_url":"https://github.com/jakesnell/quantile-risk-control/blob/HEAD/var_control/metric.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0eeb198c33d03831"}},{"code_sha256_prefix":"06c1dbc139919859","entry":"collate","repo":"jakesnell/quantile-risk-control","repo_kind":"official","path":"var_control/split.py","file_url":"https://github.com/jakesnell/quantile-risk-control/blob/HEAD/var_control/split.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"06c1dbc139919859"}},{"code_sha256_prefix":"f6472bd346be04f9","entry":"cvar_quantile_weight","repo":"jakesnell/quantile-risk-control","repo_kind":"official","path":"var_control/bounds.py","file_url":"https://github.com/jakesnell/quantile-risk-control/blob/HEAD/var_control/bounds.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f6472bd346be04f9"}},{"code_sha256_prefix":"21ed9f3405464094","entry":"fixed_split","repo":"jakesnell/quantile-risk-control","repo_kind":"official","path":"var_control/split.py","file_url":"https://github.com/jakesnell/quantile-risk-control/blob/HEAD/var_control/split.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"21ed9f3405464094"}},{"code_sha256_prefix":"51ba0556c3a09cbd","entry":"hoeffding_ucb","repo":"jakesnell/quantile-risk-control","repo_kind":"official","path":"var_control/methods.py","file_url":"https://github.com/jakesnell/quantile-risk-control/blob/HEAD/var_control/methods.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"51ba0556c3a09cbd"}},{"code_sha256_prefix":"0375f9ab3c4cf62a","entry":"interval_quantile_weight","repo":"jakesnell/quantile-risk-control","repo_kind":"official","path":"var_control/bounds.py","file_url":"https://github.com/jakesnell/quantile-risk-control/blob/HEAD/var_control/bounds.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0375f9ab3c4cf62a"}},{"code_sha256_prefix":"0c2bcecc47dcec59","entry":"is_sorted","repo":"jakesnell/quantile-risk-control","repo_kind":"official","path":"var_control/methods.py","file_url":"https://github.com/jakesnell/quantile-risk-control/blob/HEAD/var_control/methods.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0c2bcecc47dcec59"}},{"code_sha256_prefix":"642de7f4b3f7b0cc","entry":"load_features_and_ground_truth","repo":"jakesnell/quantile-risk-control","repo_kind":"official","path":"var_control/go_emotions_features.py","file_url":"https://github.com/jakesnell/quantile-risk-control/blob/HEAD/var_control/go_emotions_features.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"642de7f4b3f7b0cc"}},{"code_sha256_prefix":"22d8688f626da28e","entry":"mean_quantile_weight","repo":"jakesnell/quantile-risk-control","repo_kind":"official","path":"var_control/bounds.py","file_url":"https://github.com/jakesnell/quantile-risk-control/blob/HEAD/var_control/bounds.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"22d8688f626da28e"}},{"code_sha256_prefix":"9569954144cbd933","entry":"predict","repo":"jakesnell/quantile-risk-control","repo_kind":"official","path":"var_control/utils.py","file_url":"https://github.com/jakesnell/quantile-risk-control/blob/HEAD/var_control/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9569954144cbd933"}},{"code_sha256_prefix":"459de4a37e5bee2e","entry":"predict_batch","repo":"jakesnell/quantile-risk-control","repo_kind":"official","path":"var_control/utils.py","file_url":"https://github.com/jakesnell/quantile-risk-control/blob/HEAD/var_control/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"459de4a37e5bee2e"}},{"code_sha256_prefix":"97e0986c3fe8124e","entry":"quantile_indices","repo":"jakesnell/quantile-risk-control","repo_kind":"official","path":"var_control/utils.py","file_url":"https://github.com/jakesnell/quantile-risk-control/blob/HEAD/var_control/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"97e0986c3fe8124e"}},{"code_sha256_prefix":"16f5a97868332a84","entry":"sensitivity","repo":"jakesnell/quantile-risk-control","repo_kind":"official","path":"var_control/metric.py","file_url":"https://github.com/jakesnell/quantile-risk-control/blob/HEAD/var_control/metric.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"16f5a97868332a84"}},{"code_sha256_prefix":"41b2dcf2d6fb61d1","entry":"specificity","repo":"jakesnell/quantile-risk-control","repo_kind":"official","path":"var_control/metric.py","file_url":"https://github.com/jakesnell/quantile-risk-control/blob/HEAD/var_control/metric.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"41b2dcf2d6fb61d1"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}