{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/jointly-measuring-diversity-and-quality-in","title":"Jointly Measuring Diversity and Quality in Text Generation Models","arxiv_id":"1904.03971","date":"2019-04-08","proceeding":"NAACL 2019 6","authors":["Ehsan Montahaei","Danial Alihosseini","Mahdieh Soleymani Baghshah"],"abstract":"Text generation is an important Natural Language Processing task with various applications. Although several metrics have already been introduced to evaluate the text generation methods, each of them has its own shortcomings. The most widely used metrics such as BLEU only consider the quality of generated sentences and neglect their diversity. For example, repeatedly generation of only one high quality sentence would result in a high BLEU score. On the other hand, the more recent metric introduced to evaluate the diversity of generated texts known as Self-BLEU ignores the quality of generated texts. In this paper, we propose metrics to evaluate both the quality and diversity simultaneously by approximating the distance of the learned generative model and the real data distribution. For this purpose, we first introduce a metric that approximates this distance using n-gram based measures. Then, a feature-based measure which is based on a recent highly deep model trained on a large text corpus called BERT is introduced. Finally, for oracle training mode in which the generator's density can also be calculated, we propose to use the distance measures between the corresponding explicit distributions. Eventually, the most popular and recent text generation models are evaluated using both the existing and the proposed metrics and the preferences of the proposed metrics are determined.","url_abs":"https://arxiv.org/abs/1904.03971v2","url_pdf":"https://arxiv.org/pdf/1904.03971v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"jointly-measuring-diversity-and-quality-in","repo_url":"https://github.com/IAmS4n/TextGenerationEvaluationMetrics","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"jointly-measuring-diversity-and-quality-in","repo_url":"https://github.com/Danial-Alh/fast-bleu","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"jointly-measuring-diversity-and-quality-in","repo_url":"https://github.com/tanyuqian/progressive-generation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"diversity","task_name":"Diversity"},{"task_slug":"sentence","task_name":"Sentence"},{"task_slug":"text-generation","task_name":"Text Generation"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1904.03971","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1904.03971"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/IAmS4n/TextGenerationEvaluationMetrics","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Danial-Alh/fast-bleu","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/tanyuqian/progressive-generation","reach":null}],"summary":{"ran_fixture":1,"unverified":5},"by_repo_kind":{"official":{"samples":1,"ran":1,"repositories":1},"listed":{"samples":5,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":1,"samples":[{"code_sha256_prefix":"82fe049dd2f0dc47","entry":"calculate_frechet_distance","repo":"IAmS4n/TextGenerationEvaluationMetrics","repo_kind":"official","path":"bert_distances.py","file_url":"https://github.com/IAmS4n/TextGenerationEvaluationMetrics/blob/HEAD/bert_distances.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"82fe049dd2f0dc47"}},{"code_sha256_prefix":"ca5608be70a5b972","entry":"closest_ref_length","repo":"Danial-Alh/fast-bleu","repo_kind":"listed","path":"old_metrics/bleu-old.py","file_url":"https://github.com/Danial-Alh/fast-bleu/blob/HEAD/old_metrics/bleu-old.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ca5608be70a5b972"}},{"code_sha256_prefix":"8b5bec45d74d3206","entry":"corpus_bleu","repo":"Danial-Alh/fast-bleu","repo_kind":"listed","path":"old_metrics/bleu-old.py","file_url":"https://github.com/Danial-Alh/fast-bleu/blob/HEAD/old_metrics/bleu-old.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8b5bec45d74d3206"}},{"code_sha256_prefix":"6b596e04b33971a0","entry":"get_ngrams","repo":"Danial-Alh/fast-bleu","repo_kind":"listed","path":"old_metrics/utils.py","file_url":"https://github.com/Danial-Alh/fast-bleu/blob/HEAD/old_metrics/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6b596e04b33971a0"}},{"code_sha256_prefix":"30fb058c70150725","entry":"modified_precision","repo":"Danial-Alh/fast-bleu","repo_kind":"listed","path":"old_metrics/bleu-old.py","file_url":"https://github.com/Danial-Alh/fast-bleu/blob/HEAD/old_metrics/bleu-old.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"30fb058c70150725"}},{"code_sha256_prefix":"ba61bc7fc6c3ca92","entry":"tokenize","repo":"Danial-Alh/fast-bleu","repo_kind":"listed","path":"old_metrics/utils.py","file_url":"https://github.com/Danial-Alh/fast-bleu/blob/HEAD/old_metrics/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ba61bc7fc6c3ca92"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}