{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/reliable-fidelity-and-diversity-metrics-for","title":"Reliable Fidelity and Diversity Metrics for Generative Models","arxiv_id":"2002.09797","date":"2020-02-23","proceeding":"ICML 2020 1","authors":["Muhammad Ferjad Naeem","Seong Joon Oh","Youngjung Uh","Yunjey Choi","Jaejun Yoo"],"abstract":"Devising indicative evaluation metrics for the image generation task remains an open problem. The most widely used metric for measuring the similarity between real and generated images has been the Fr\\'echet Inception Distance (FID) score. Because it does not differentiate the fidelity and diversity aspects of the generated images, recent papers have introduced variants of precision and recall metrics to diagnose those properties separately. In this paper, we show that even the latest version of the precision and recall metrics are not reliable yet. For example, they fail to detect the match between two identical distributions, they are not robust against outliers, and the evaluation hyperparameters are selected arbitrarily. We propose density and coverage metrics that solve the above issues. We analytically and experimentally show that density and coverage provide more interpretable and reliable signals for practitioners than the existing metrics. Code: https://github.com/clovaai/generative-evaluation-prdc.","url_abs":"https://arxiv.org/abs/2002.09797v2","url_pdf":"https://arxiv.org/pdf/2002.09797v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"reliable-fidelity-and-diversity-metrics-for","repo_url":"https://github.com/clovaai/generative-evaluation-prdc","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":null},{"paper_slug":"reliable-fidelity-and-diversity-metrics-for","repo_url":"https://github.com/kozistr/gan-metrics","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"reliable-fidelity-and-diversity-metrics-for","repo_url":"https://github.com/sonycslparis/audio-metrics","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"diversity","task_name":"Diversity"},{"task_slug":"image-generation","task_name":"Image Generation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2002.09797","atlas_url":"https://app.syntology.ai/?focus=2002.09797","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2002.09797"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/clovaai/generative-evaluation-prdc","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sonycslparis/audio-metrics","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kozistr/gan-metrics","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_honours":1,"ran_draft_wrong":2,"ran":2,"unverified":4},"by_repo_kind":{"official":{"samples":4,"ran":0,"repositories":1},"listed":{"samples":3,"ran":3,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":3,"samples":[{"code_sha256_prefix":"259ec7338be8a457","entry":"compute_pairwise_distance","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_honours","verification_level":2,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"259ec7338be8a457"}},{"code_sha256_prefix":"547662aecde999ed","entry":"get_kth_value","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"547662aecde999ed"}},{"code_sha256_prefix":"99ef435f17f2d479","entry":"is_valid_batch_size","repo":"kozistr/gan-metrics","repo_kind":"listed","path":"metrics/utils.py","file_url":"https://github.com/kozistr/gan-metrics/blob/HEAD/metrics/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"99ef435f17f2d479"}},{"code_sha256_prefix":"11beb5b7654463dd","entry":"is_valid_shape","repo":"kozistr/gan-metrics","repo_kind":"listed","path":"metrics/utils.py","file_url":"https://github.com/kozistr/gan-metrics/blob/HEAD/metrics/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"11beb5b7654463dd"}},{"code_sha256_prefix":"9ece347078e1df5e","entry":"prdc","repo":"sonycslparis/audio-metrics","repo_kind":"listed","path":"src/audio_metrics/metrics/prdc.py","file_url":"https://github.com/sonycslparis/audio-metrics/blob/HEAD/src/audio_metrics/metrics/prdc.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"9ece347078e1df5e"}},{"code_sha256_prefix":"a408e16ed43950a5","entry":"compute_nearest_neighbour_distances","repo":"clovaai/generative-evaluation-prdc","repo_kind":"official","path":"prdc/prdc.py","file_url":"https://github.com/clovaai/generative-evaluation-prdc/blob/HEAD/prdc/prdc.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a408e16ed43950a5"}},{"code_sha256_prefix":"0c0553c682e66b85","entry":"compute_pairwise_distance","repo":"clovaai/generative-evaluation-prdc","repo_kind":"official","path":"prdc/prdc.py","file_url":"https://github.com/clovaai/generative-evaluation-prdc/blob/HEAD/prdc/prdc.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0c0553c682e66b85"}},{"code_sha256_prefix":"ea3158807a7ddbf6","entry":"compute_prdc","repo":"clovaai/generative-evaluation-prdc","repo_kind":"official","path":"prdc/prdc.py","file_url":"https://github.com/clovaai/generative-evaluation-prdc/blob/HEAD/prdc/prdc.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ea3158807a7ddbf6"}},{"code_sha256_prefix":"366a31e98268c0e3","entry":"get_kth_value","repo":"clovaai/generative-evaluation-prdc","repo_kind":"official","path":"prdc/prdc.py","file_url":"https://github.com/clovaai/generative-evaluation-prdc/blob/HEAD/prdc/prdc.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"366a31e98268c0e3"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}