{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/confidence-calibration-of-classifiers-with","title":"Confidence Calibration of Classifiers with Many Classes","arxiv_id":"2411.02988","date":"2024-11-05","proceeding":null,"authors":["Adrien LeCoz","Stéphane Herbin","Faouzi Adjed"],"abstract":"For classification models based on neural networks, the maximum predicted class probability is often used as a confidence score. This score rarely predicts well the probability of making a correct prediction and requires a post-processing calibration step. However, many confidence calibration methods fail for problems with many classes. To address this issue, we transform the problem of calibrating a multiclass classifier into calibrating a single surrogate binary classifier. This approach allows for more efficient use of standard calibration methods. We evaluate our approach on numerous neural networks used for image or text classification and show that it significantly enhances existing calibration methods.","url_abs":"https://arxiv.org/abs/2411.02988v2","url_pdf":"https://arxiv.org/pdf/2411.02988v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"confidence-calibration-of-classifiers-with","repo_url":"https://github.com/allglc/tva-calibration","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"text-classification","task_name":"Text Classification"},{"task_slug":"text-classification-1","task_name":"text-classification"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2411.02988","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2411.02988"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/allglc/tva-calibration","reach":null},{"provenance":"deterministic:regex_extraction","url":"https://github.com/gpleiss/temperature_scaling","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/dirichletcal/experiments_dnn","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/torrvision/focal_","reach":{"status":"gone","observed_at":"2026-09-17","how":"tree_404+repo_404"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/lifan-yuan/PLMCalibration","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/mominabbass/LinC","reach":{"status":"ok"}}],"summary":{"ran_fixture":3,"ran_draft_wrong":2,"unverified":13},"by_repo_kind":{"official":{"samples":3,"ran":3,"repositories":1},"found_in_text":{"samples":15,"ran":2,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":9,"samples":[{"code_sha256_prefix":"97600a3445a5f8c5","entry":"AUROC","repo":"allglc/tva-calibration","repo_kind":"official","path":"evaluation.py","file_url":"https://github.com/allglc/tva-calibration/blob/HEAD/evaluation.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"97600a3445a5f8c5"}},{"code_sha256_prefix":"a3002e7b9386d6dd","entry":"ECE_calc","repo":"allglc/tva-calibration","repo_kind":"official","path":"evaluation.py","file_url":"https://github.com/allglc/tva-calibration/blob/HEAD/evaluation.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a3002e7b9386d6dd"}},{"code_sha256_prefix":"47774a4ca35cf0e3","entry":"ece_equal_interval","repo":"lifan-yuan/PLMCalibration","repo_kind":"found_in_text","path":"metric.py","file_url":"https://github.com/lifan-yuan/PLMCalibration/blob/HEAD/metric.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"47774a4ca35cf0e3"}},{"code_sha256_prefix":"4427a224ec9b8f4f","entry":"ece_equal_mass","repo":"lifan-yuan/PLMCalibration","repo_kind":"found_in_text","path":"metric.py","file_url":"https://github.com/lifan-yuan/PLMCalibration/blob/HEAD/metric.py","link_basis":"plan_row","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4427a224ec9b8f4f"}},{"code_sha256_prefix":"96ab79467959b134","entry":"fit_scaling_model","repo":"allglc/tva-calibration","repo_kind":"official","path":"calibrators.py","file_url":"https://github.com/allglc/tva-calibration/blob/HEAD/calibrators.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"96ab79467959b134"}},{"code_sha256_prefix":"e775486237d19cbf","entry":"chunk_size_helper","repo":"mominabbass/LinC","repo_kind":"found_in_text","path":"utils.py","file_url":"https://github.com/mominabbass/LinC/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e775486237d19cbf"}},{"code_sha256_prefix":"e115d79cf282ebb9","entry":"compute_hidden","repo":"lifan-yuan/PLMCalibration","repo_kind":"found_in_text","path":"utils/calibration_methods.py","file_url":"https://github.com/lifan-yuan/PLMCalibration/blob/HEAD/utils/calibration_methods.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e115d79cf282ebb9"}},{"code_sha256_prefix":"4db3f48d30d8c5af","entry":"eval_accuracy","repo":"mominabbass/LinC","repo_kind":"found_in_text","path":"run_classification.py","file_url":"https://github.com/mominabbass/LinC/blob/HEAD/run_classification.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4db3f48d30d8c5af"}},{"code_sha256_prefix":"e2ecbd7ff12e83d5","entry":"evaluate","repo":"lifan-yuan/PLMCalibration","repo_kind":"found_in_text","path":"prompt-dynamics.py","file_url":"https://github.com/lifan-yuan/PLMCalibration/blob/HEAD/prompt-dynamics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e2ecbd7ff12e83d5"}},{"code_sha256_prefix":"5439c7c6310eaacf","entry":"evaluation","repo":"lifan-yuan/PLMCalibration","repo_kind":"found_in_text","path":"prompt-ood.py","file_url":"https://github.com/lifan-yuan/PLMCalibration/blob/HEAD/prompt-ood.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5439c7c6310eaacf"}},{"code_sha256_prefix":"85faff7a2af04107","entry":"evaluation","repo":"lifan-yuan/PLMCalibration","repo_kind":"found_in_text","path":"prompt-emergent.py","file_url":"https://github.com/lifan-yuan/PLMCalibration/blob/HEAD/prompt-emergent.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"85faff7a2af04107"}},{"code_sha256_prefix":"dc60c15158733e34","entry":"expected_calibration_error","repo":"mominabbass/LinC","repo_kind":"found_in_text","path":"utils.py","file_url":"https://github.com/mominabbass/LinC/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"dc60c15158733e34"}},{"code_sha256_prefix":"77347b697708c203","entry":"get_label_probs","repo":"mominabbass/LinC","repo_kind":"found_in_text","path":"run_classification.py","file_url":"https://github.com/mominabbass/LinC/blob/HEAD/run_classification.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"77347b697708c203"}},{"code_sha256_prefix":"f1916eb36c894684","entry":"get_p_content_free","repo":"mominabbass/LinC","repo_kind":"found_in_text","path":"run_classification.py","file_url":"https://github.com/mominabbass/LinC/blob/HEAD/run_classification.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f1916eb36c894684"}},{"code_sha256_prefix":"2b34620ed2edab90","entry":"preprocess","repo":"lifan-yuan/PLMCalibration","repo_kind":"found_in_text","path":"train-bow-tf_idf.py","file_url":"https://github.com/lifan-yuan/PLMCalibration/blob/HEAD/train-bow-tf_idf.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2b34620ed2edab90"}},{"code_sha256_prefix":"c6b46764733c312e","entry":"random_sampling","repo":"mominabbass/LinC","repo_kind":"found_in_text","path":"utils.py","file_url":"https://github.com/mominabbass/LinC/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c6b46764733c312e"}},{"code_sha256_prefix":"6824958e0d4a8b02","entry":"vectorize","repo":"lifan-yuan/PLMCalibration","repo_kind":"found_in_text","path":"train-bow-tf_idf.py","file_url":"https://github.com/lifan-yuan/PLMCalibration/blob/HEAD/train-bow-tf_idf.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6824958e0d4a8b02"}},{"code_sha256_prefix":"69772baf34349d01","entry":"wrap_feature_based_testloader","repo":"lifan-yuan/PLMCalibration","repo_kind":"found_in_text","path":"utils/calibration_methods.py","file_url":"https://github.com/lifan-yuan/PLMCalibration/blob/HEAD/utils/calibration_methods.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"69772baf34349d01"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}