{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/visual-correspondence-based-explanations","title":"Visual correspondence-based explanations improve AI robustness and human-AI team accuracy","arxiv_id":"2208.00780","date":"2022-07-26","proceeding":null,"authors":["Giang Nguyen","Mohammad Reza Taesiri","Anh Nguyen"],"abstract":"Explaining artificial intelligence (AI) predictions is increasingly important and even imperative in many high-stakes applications where humans are the ultimate decision-makers. In this work, we propose two novel architectures of self-interpretable image classifiers that first explain, and then predict (as opposed to post-hoc explanations) by harnessing the visual correspondences between a query image and exemplars. Our models consistently improve (by 1 to 4 points) on out-of-distribution (OOD) datasets while performing marginally worse (by 1 to 2 points) on in-distribution tests than ResNet-50 and a $k$-nearest neighbor classifier (kNN). Via a large-scale, human study on ImageNet and CUB, our correspondence-based explanations are found to be more useful to users than kNN explanations. Our explanations help users more accurately reject AI's wrong decisions than all other tested methods. Interestingly, for the first time, we show that it is possible to achieve complementary human-AI team accuracy (i.e., that is higher than either AI-alone or human-alone), in ImageNet and CUB image classification tasks.","url_abs":"https://arxiv.org/abs/2208.00780v5","url_pdf":"https://arxiv.org/pdf/2208.00780v5.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"visual-correspondence-based-explanations","repo_url":"https://github.com/anguyen8/visual-correspondence-xai","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"adversarial-robustness","task_name":"Adversarial Robustness"},{"task_slug":"fine-grained-image-classification","task_name":"Fine-Grained Image Classification"},{"task_slug":"image-classification","task_name":"Image Classification"},{"task_slug":"image-classification","task_name":"image-classification"}],"methods":[{"method_slug":"average-pooling","method_name":"Average Pooling"},{"method_slug":"chm","method_name":"CHM"},{"method_slug":"convolution","method_name":"Convolution"},{"method_slug":"global-average-pooling","method_name":"Global Average Pooling"},{"method_slug":"kaiming-initialization","method_name":"Kaiming Initialization"},{"method_slug":"max-pooling","method_name":"Max Pooling"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2208.00780","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2208.00780"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/anguyen8/visual-correspondence-xai","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":2,"unverified":2},"by_repo_kind":{"official":{"samples":4,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"d9def42110729a85","entry":"conv1x1","repo":"anguyen8/visual-correspondence-xai","repo_kind":"official","path":"EMD-Corr/src/cub200_features.py","file_url":"https://github.com/anguyen8/visual-correspondence-xai/blob/HEAD/EMD-Corr/src/cub200_features.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d9def42110729a85"}},{"code_sha256_prefix":"fac5364e2f53c6db","entry":"conv3x3","repo":"anguyen8/visual-correspondence-xai","repo_kind":"official","path":"EMD-Corr/src/cub200_features.py","file_url":"https://github.com/anguyen8/visual-correspondence-xai/blob/HEAD/EMD-Corr/src/cub200_features.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fac5364e2f53c6db"}},{"code_sha256_prefix":"9bd8d03f1212f6ef","entry":"get_resnet50_features","repo":"anguyen8/visual-correspondence-xai","repo_kind":"official","path":"EMD-Corr/src/cub200_features.py","file_url":"https://github.com/anguyen8/visual-correspondence-xai/blob/HEAD/EMD-Corr/src/cub200_features.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9bd8d03f1212f6ef"}},{"code_sha256_prefix":"9355a82af5a3b83b","entry":"resnet50_features","repo":"anguyen8/visual-correspondence-xai","repo_kind":"official","path":"ResNet-50/CUB-iNaturalist/FeatureExtractors.py","file_url":"https://github.com/anguyen8/visual-correspondence-xai/blob/HEAD/ResNet-50/CUB-iNaturalist/FeatureExtractors.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9355a82af5a3b83b"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}