{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/non-parametric-calibration-for-classification","title":"Non-Parametric Calibration for Classification","arxiv_id":"1906.04933","date":"2019-06-12","proceeding":null,"authors":["Jonathan Wenger","Hedvig Kjellström","Rudolph Triebel"],"abstract":"Many applications of classification methods not only require high accuracy but also reliable estimation of predictive uncertainty. However, while many current classification frameworks, in particular deep neural networks, achieve high accuracy, they tend to incorrectly estimate uncertainty. In this paper, we propose a method that adjusts the confidence estimates of a general classifier such that they approach the probability of classifying correctly. In contrast to existing approaches, our calibration method employs a non-parametric representation using a latent Gaussian process, and is specifically designed for multi-class classification. It can be applied to any classifier that outputs confidence estimates and is not limited to neural networks. We also provide a theoretical analysis regarding the over- and underconfidence of a classifier and its relationship to calibration, as well as an empirical outlook for calibrated active learning. In experiments we show the universally strong performance of our method across different classifiers and benchmark data sets, in particular for state-of-the art neural network architectures.","url_abs":"https://arxiv.org/abs/1906.04933v3","url_pdf":"https://arxiv.org/pdf/1906.04933v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"non-parametric-calibration-for-classification","repo_url":"https://github.com/JonathanWenger/pycalib","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"active-learning","task_name":"Active Learning"},{"task_slug":"classification-1","task_name":"Classification"},{"task_slug":"classification","task_name":"General Classification"},{"task_slug":"multi-class-classification","task_name":"Multi-class Classification"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1906.04933","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1906.04933"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/JonathanWenger/pycalib","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":1,"unverified":8},"by_repo_kind":{"official":{"samples":9,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"fac5364e2f53c6db","entry":"conv3x3","repo":"JonathanWenger/pycalib","repo_kind":"official","path":"pycalib/models/cifar100/preresnet.py","file_url":"https://github.com/JonathanWenger/pycalib/blob/HEAD/pycalib/models/cifar100/preresnet.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fac5364e2f53c6db"}},{"code_sha256_prefix":"dde454b6ff5af4f7","entry":"accuracy","repo":"JonathanWenger/pycalib","repo_kind":"official","path":"pycalib/scoring.py","file_url":"https://github.com/JonathanWenger/pycalib/blob/HEAD/pycalib/scoring.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dde454b6ff5af4f7"}},{"code_sha256_prefix":"187e9f7c92123922","entry":"error","repo":"JonathanWenger/pycalib","repo_kind":"official","path":"pycalib/scoring.py","file_url":"https://github.com/JonathanWenger/pycalib/blob/HEAD/pycalib/scoring.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"187e9f7c92123922"}},{"code_sha256_prefix":"f47b221c34868676","entry":"figure","repo":"JonathanWenger/pycalib","repo_kind":"official","path":"pycalib/texfig.py","file_url":"https://github.com/JonathanWenger/pycalib/blob/HEAD/pycalib/texfig.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f47b221c34868676"}},{"code_sha256_prefix":"069299b0a43b9f5c","entry":"odds_correctness","repo":"JonathanWenger/pycalib","repo_kind":"official","path":"pycalib/scoring.py","file_url":"https://github.com/JonathanWenger/pycalib/blob/HEAD/pycalib/scoring.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"069299b0a43b9f5c"}},{"code_sha256_prefix":"ca2137db7d7be190","entry":"query_margin","repo":"JonathanWenger/pycalib","repo_kind":"official","path":"pycalib/active_learning.py","file_url":"https://github.com/JonathanWenger/pycalib/blob/HEAD/pycalib/active_learning.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ca2137db7d7be190"}},{"code_sha256_prefix":"ccb3d107ee45500b","entry":"query_norm_entropy","repo":"JonathanWenger/pycalib","repo_kind":"official","path":"pycalib/active_learning.py","file_url":"https://github.com/JonathanWenger/pycalib/blob/HEAD/pycalib/active_learning.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ccb3d107ee45500b"}},{"code_sha256_prefix":"94daa890a509337f","entry":"query_uncertainty","repo":"JonathanWenger/pycalib","repo_kind":"official","path":"pycalib/active_learning.py","file_url":"https://github.com/JonathanWenger/pycalib/blob/HEAD/pycalib/active_learning.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"94daa890a509337f"}},{"code_sha256_prefix":"1e7b15ce5df2da9b","entry":"subplots","repo":"JonathanWenger/pycalib","repo_kind":"official","path":"pycalib/texfig.py","file_url":"https://github.com/JonathanWenger/pycalib/blob/HEAD/pycalib/texfig.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1e7b15ce5df2da9b"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}