{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/evaluating-model-calibration-in","title":"Evaluating model calibration in classification","arxiv_id":"1902.06977","date":"2019-02-19","proceeding":null,"authors":["Juozas Vaicenavicius","David Widmann","Carl Andersson","Fredrik Lindsten","Jacob Roll","Thomas B. Schön"],"abstract":"Probabilistic classifiers output a probability distribution on target classes\nrather than just a class prediction. Besides providing a clear separation of\nprediction and decision making, the main advantage of probabilistic models is\ntheir ability to represent uncertainty about predictions. In safety-critical\napplications, it is pivotal for a model to possess an adequate sense of\nuncertainty, which for probabilistic classifiers translates into outputting\nprobability distributions that are consistent with the empirical frequencies\nobserved from realized outcomes. A classifier with such a property is called\ncalibrated. In this work, we develop a general theoretical calibration\nevaluation framework grounded in probability theory, and point out subtleties\npresent in model calibration evaluation that lead to refined interpretations of\nexisting evaluation techniques. Lastly, we propose new ways to quantify and\nvisualize miscalibration in probabilistic classification, including novel\nmultidimensional reliability diagrams.","url_abs":"http://arxiv.org/abs/1902.06977v1","url_pdf":"http://arxiv.org/pdf/1902.06977v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"evaluating-model-calibration-in","repo_url":"https://github.com/uu-sml/calibration","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"none","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"classification-1","task_name":"Classification"},{"task_slug":"decision-making","task_name":"Decision Making"},{"task_slug":"classification","task_name":"General Classification"},{"task_slug":"model","task_name":"model"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1902.06977","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1902.06977"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/uu-sml/calibration","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":8},"by_repo_kind":{"official":{"samples":8,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"6c403355b95b1151","entry":"consistent_targets","repo":"uu-sml/calibration","repo_kind":"official","path":"calibration/sample/consistent.py","file_url":"https://github.com/uu-sml/calibration/blob/HEAD/calibration/sample/consistent.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6c403355b95b1151"}},{"code_sha256_prefix":"2971e1e1803b3bb6","entry":"group_lens","repo":"uu-sml/calibration","repo_kind":"official","path":"calibration/lenses/group.py","file_url":"https://github.com/uu-sml/calibration/blob/HEAD/calibration/lenses/group.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2971e1e1803b3bb6"}},{"code_sha256_prefix":"d9cccedb054bf4d1","entry":"logarithmic_score","repo":"uu-sml/calibration","repo_kind":"official","path":"calibration/scores/logarithmic.py","file_url":"https://github.com/uu-sml/calibration/blob/HEAD/calibration/scores/logarithmic.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d9cccedb054bf4d1"}},{"code_sha256_prefix":"f052a2430375201f","entry":"maximum_lens","repo":"uu-sml/calibration","repo_kind":"official","path":"calibration/lenses/max.py","file_url":"https://github.com/uu-sml/calibration/blob/HEAD/calibration/lenses/max.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f052a2430375201f"}},{"code_sha256_prefix":"4f7a1cc8b67a6813","entry":"quadratic_score","repo":"uu-sml/calibration","repo_kind":"official","path":"calibration/scores/quadratic.py","file_url":"https://github.com/uu-sml/calibration/blob/HEAD/calibration/scores/quadratic.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4f7a1cc8b67a6813"}},{"code_sha256_prefix":"fc42a6bf6e5ed961","entry":"resample_stats","repo":"uu-sml/calibration","repo_kind":"official","path":"calibration/sample/resample.py","file_url":"https://github.com/uu-sml/calibration/blob/HEAD/calibration/sample/resample.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fc42a6bf6e5ed961"}},{"code_sha256_prefix":"7774e83d22de764f","entry":"spherical_score","repo":"uu-sml/calibration","repo_kind":"official","path":"calibration/scores/spherical.py","file_url":"https://github.com/uu-sml/calibration/blob/HEAD/calibration/scores/spherical.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7774e83d22de764f"}},{"code_sha256_prefix":"01608ff38bf20a2e","entry":"top2_lens","repo":"uu-sml/calibration","repo_kind":"official","path":"calibration/lenses/top2.py","file_url":"https://github.com/uu-sml/calibration/blob/HEAD/calibration/lenses/top2.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"01608ff38bf20a2e"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}