{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/a-causal-framework-for-evaluating-deferring","title":"A Causal Framework for Evaluating Deferring Systems","arxiv_id":"2405.18902","date":"2024-05-29","proceeding":null,"authors":["Filippo Palomba","Andrea Pugnana","José Manuel Alvarez","Salvatore Ruggieri"],"abstract":"Deferring systems extend supervised Machine Learning (ML) models with the possibility to defer predictions to human experts. However, evaluating the impact of a deferring strategy on system accuracy is still an overlooked area. This paper fills this gap by evaluating deferring systems through a causal lens. We link the potential outcomes framework for causal inference with deferring systems, which allows to identify the causal impact of the deferring strategy on predictive accuracy. We distinguish two scenarios. In the first one, we have access to both the human and ML model predictions for the deferred instances. Here, we can identify the individual causal effects for deferred instances and the aggregates of them. In the second one, only human predictions are available for the deferred instances. Here, we can resort to regression discontinuity designs to estimate a local causal effect. We evaluate our approach on synthetic and real datasets for seven deferring systems from the literature.","url_abs":"https://arxiv.org/abs/2405.18902v2","url_pdf":"https://arxiv.org/pdf/2405.18902v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"a-causal-framework-for-evaluating-deferring","repo_url":"https://github.com/andrepugni/pods","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"causal-inference","task_name":"Causal Inference"}],"methods":[{"method_slug":"causal-inference","method_name":"Causal inference"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2405.18902","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.18902"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/andrepugni/pods","reach":{"status":"ok"}}],"summary":{"ran":4,"unverified":3},"by_repo_kind":{"official":{"samples":7,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":7,"samples":[{"code_sha256_prefix":"b50bd40d9c6f9f82","entry":"compute_classification_metrics","repo":"andrepugni/pods","repo_kind":"official","path":"src/metrics.py","file_url":"https://github.com/andrepugni/pods/blob/HEAD/src/metrics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b50bd40d9c6f9f82"}},{"code_sha256_prefix":"5a93887cd0f16444","entry":"compute_deferral_metrics","repo":"andrepugni/pods","repo_kind":"official","path":"src/metrics.py","file_url":"https://github.com/andrepugni/pods/blob/HEAD/src/metrics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5a93887cd0f16444"}},{"code_sha256_prefix":"6b099063d80dc67f","entry":"mobilenet_v2","repo":"andrepugni/pods","repo_kind":"official","path":"src/networks.py","file_url":"https://github.com/andrepugni/pods/blob/HEAD/src/networks.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6b099063d80dc67f"}},{"code_sha256_prefix":"12f9e625916d84cf","entry":"weighted_cross_entropy_loss","repo":"andrepugni/pods","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/andrepugni/pods/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"12f9e625916d84cf"}},{"code_sha256_prefix":"7e3bc1bfab694f64","entry":"compute_coverage_v_acc_curve","repo":"andrepugni/pods","repo_kind":"official","path":"src/metrics.py","file_url":"https://github.com/andrepugni/pods/blob/HEAD/src/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7e3bc1bfab694f64"}},{"code_sha256_prefix":"165b152440beb6b4","entry":"train_deferral_single_model","repo":"andrepugni/pods","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/andrepugni/pods/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"165b152440beb6b4"}},{"code_sha256_prefix":"c7d16d62b7ab8c9c","entry":"train_single_model","repo":"andrepugni/pods","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/andrepugni/pods/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c7d16d62b7ab8c9c"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}