{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/evaluation-criteria-for-instance-based","title":"Evaluation of Similarity-based Explanations","arxiv_id":"2006.04528","date":"2020-06-08","proceeding":"ICLR 2021 1","authors":["Kazuaki Hanawa","Sho Yokoi","Satoshi Hara","Kentaro Inui"],"abstract":"Explaining the predictions made by complex machine learning models helps users to understand and accept the predicted outputs with confidence. One promising way is to use similarity-based explanation that provides similar instances as evidence to support model predictions. Several relevance metrics are used for this purpose. In this study, we investigated relevance metrics that can provide reasonable explanations to users. Specifically, we adopted three tests to evaluate whether the relevance metrics satisfy the minimal requirements for similarity-based explanation. Our experiments revealed that the cosine similarity of the gradients of the loss performs best, which would be a recommended choice in practice. In addition, we showed that some metrics perform poorly in our tests and analyzed the reasons of their failure. We expect our insights to help practitioners in selecting appropriate relevance metrics and also aid further researches for designing better relevance metrics for explanations.","url_abs":"https://arxiv.org/abs/2006.04528v2","url_pdf":"https://arxiv.org/pdf/2006.04528v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"evaluation-criteria-for-instance-based","repo_url":"https://github.com/k-hanawa/criteria_for_instance_based_explanation","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"evaluation-criteria-for-instance-based","repo_url":"https://github.com/rbitr/pytkml","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2006.04528","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2006.04528"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/rbitr/pytkml","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/k-hanawa/criteria_for_instance_based_explanation","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":16},"by_repo_kind":{"official":{"samples":9,"ran":0,"repositories":1},"listed":{"samples":7,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"b4fab1a9ec540acb","entry":"RandomSubsetLoader","repo":"rbitr/pytkml","repo_kind":"listed","path":"pytkml/core/dataloaders.py","file_url":"https://github.com/rbitr/pytkml/blob/HEAD/pytkml/core/dataloaders.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b4fab1a9ec540acb"}},{"code_sha256_prefix":"95ddf503955b614b","entry":"argmax_reducer","repo":"rbitr/pytkml","repo_kind":"listed","path":"pytkml/core/processors.py","file_url":"https://github.com/rbitr/pytkml/blob/HEAD/pytkml/core/processors.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"95ddf503955b614b"}},{"code_sha256_prefix":"e5f4ff718f6859ca","entry":"arrayToBase64IM","repo":"rbitr/pytkml","repo_kind":"listed","path":"pytkml/misc/logging.py","file_url":"https://github.com/rbitr/pytkml/blob/HEAD/pytkml/misc/logging.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e5f4ff718f6859ca"}},{"code_sha256_prefix":"fe4a8d068a98b3cb","entry":"cossim","repo":"k-hanawa/criteria_for_instance_based_explanation","repo_kind":"official","path":"src/metrics.py","file_url":"https://github.com/k-hanawa/criteria_for_instance_based_explanation/blob/HEAD/src/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fe4a8d068a98b3cb"}},{"code_sha256_prefix":"a1a1dfb1820e0b9f","entry":"first_element_reducer","repo":"rbitr/pytkml","repo_kind":"listed","path":"pytkml/core/processors.py","file_url":"https://github.com/rbitr/pytkml/blob/HEAD/pytkml/core/processors.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a1a1dfb1820e0b9f"}},{"code_sha256_prefix":"1e772e343910909d","entry":"first_element_reducerInt","repo":"rbitr/pytkml","repo_kind":"listed","path":"pytkml/core/processors.py","file_url":"https://github.com/rbitr/pytkml/blob/HEAD/pytkml/core/processors.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1e772e343910909d"}},{"code_sha256_prefix":"e19548528043a985","entry":"flatten","repo":"k-hanawa/criteria_for_instance_based_explanation","repo_kind":"official","path":"src/torch/mobilenetv2.py","file_url":"https://github.com/k-hanawa/criteria_for_instance_based_explanation/blob/HEAD/src/torch/mobilenetv2.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e19548528043a985"}},{"code_sha256_prefix":"d79d6438697e1ed3","entry":"get_cifar10_data","repo":"k-hanawa/criteria_for_instance_based_explanation","repo_kind":"official","path":"src/get_dataset.py","file_url":"https://github.com/k-hanawa/criteria_for_instance_based_explanation/blob/HEAD/src/get_dataset.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d79d6438697e1ed3"}},{"code_sha256_prefix":"6b8474e9877e3a4b","entry":"get_entries","repo":"rbitr/pytkml","repo_kind":"listed","path":"pytkml/misc/logging.py","file_url":"https://github.com/rbitr/pytkml/blob/HEAD/pytkml/misc/logging.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6b8474e9877e3a4b"}},{"code_sha256_prefix":"806006ec6b9a7c95","entry":"get_mnist_data","repo":"k-hanawa/criteria_for_instance_based_explanation","repo_kind":"official","path":"src/get_dataset.py","file_url":"https://github.com/k-hanawa/criteria_for_instance_based_explanation/blob/HEAD/src/get_dataset.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"806006ec6b9a7c95"}},{"code_sha256_prefix":"fbab2ae0bfd9ace7","entry":"get_trec_data","repo":"k-hanawa/criteria_for_instance_based_explanation","repo_kind":"official","path":"src/get_dataset.py","file_url":"https://github.com/k-hanawa/criteria_for_instance_based_explanation/blob/HEAD/src/get_dataset.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fbab2ae0bfd9ace7"}},{"code_sha256_prefix":"1918e0892fcf567b","entry":"influence_dot","repo":"k-hanawa/criteria_for_instance_based_explanation","repo_kind":"official","path":"src/metrics.py","file_url":"https://github.com/k-hanawa/criteria_for_instance_based_explanation/blob/HEAD/src/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1918e0892fcf567b"}},{"code_sha256_prefix":"5559c3b944f881f0","entry":"l2sim","repo":"k-hanawa/criteria_for_instance_based_explanation","repo_kind":"official","path":"src/metrics.py","file_url":"https://github.com/k-hanawa/criteria_for_instance_based_explanation/blob/HEAD/src/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5559c3b944f881f0"}},{"code_sha256_prefix":"9967ce3a4eaa3100","entry":"make_dataloaders","repo":"k-hanawa/criteria_for_instance_based_explanation","repo_kind":"official","path":"src/torch/mobilenetv2.py","file_url":"https://github.com/k-hanawa/criteria_for_instance_based_explanation/blob/HEAD/src/torch/mobilenetv2.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9967ce3a4eaa3100"}},{"code_sha256_prefix":"d1bf41dee7f45a7f","entry":"mobilenet_v2","repo":"k-hanawa/criteria_for_instance_based_explanation","repo_kind":"official","path":"src/torch/mobilenetv2.py","file_url":"https://github.com/k-hanawa/criteria_for_instance_based_explanation/blob/HEAD/src/torch/mobilenetv2.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d1bf41dee7f45a7f"}},{"code_sha256_prefix":"f6502b235ea0f868","entry":"renderLogAsHTML","repo":"rbitr/pytkml","repo_kind":"listed","path":"pytkml/misc/logging.py","file_url":"https://github.com/rbitr/pytkml/blob/HEAD/pytkml/misc/logging.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f6502b235ea0f868"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}