{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/the-curious-decline-of-linguistic-diversity","title":"The Curious Decline of Linguistic Diversity: Training Language Models on Synthetic Text","arxiv_id":"2311.09807","date":"2023-11-16","proceeding":null,"authors":["Yanzhu Guo","Guokan Shang","Michalis Vazirgiannis","Chloé Clavel"],"abstract":"This study investigates the consequences of training language models on synthetic data generated by their predecessors, an increasingly prevalent practice given the prominence of powerful generative models. Diverging from the usual emphasis on performance metrics, we focus on the impact of this training methodology on linguistic diversity, especially when conducted recursively over time. To assess this, we adapt and develop a set of novel metrics targeting lexical, syntactic, and semantic diversity, applying them in recursive finetuning experiments across various natural language generation tasks in English. Our findings reveal a consistent decrease in the diversity of the model outputs through successive iterations, especially remarkable for tasks demanding high levels of creativity. This trend underscores the potential risks of training language models on synthetic text, particularly concerning the preservation of linguistic richness. Our study highlights the need for careful consideration of the long-term effects of such training approaches on the linguistic capabilities of language models.","url_abs":"https://arxiv.org/abs/2311.09807v2","url_pdf":"https://arxiv.org/pdf/2311.09807v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"the-curious-decline-of-linguistic-diversity","repo_url":"https://github.com/yanzhuguo/linguistic-diversity","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"CC0-1.0"}}],"tasks":[{"task_slug":"diversity","task_name":"Diversity"},{"task_slug":"text-generation","task_name":"Text Generation"}],"methods":[{"method_slug":"focus","method_name":"Focus"},{"method_slug":"set","method_name":"SET"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2311.09807","atlas_url":"https://app.syntology.ai/?focus=2311.09807","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.09807"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yanzhuguo/linguistic-diversity","reach":{"status":"ok","spdx":"CC0-1.0"}}],"summary":{"ran":3},"by_repo_kind":{"official":{"samples":3,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"9085f68964b41f96","entry":"create_graph","repo":"yanzhuguo/linguistic-diversity","repo_kind":"official","path":"syntactic_diversity.py","file_url":"https://github.com/yanzhuguo/linguistic-diversity/blob/HEAD/syntactic_diversity.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"CC0-1.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9085f68964b41f96"}},{"code_sha256_prefix":"225f4d9f0906b5a9","entry":"mean_confidence_interval","repo":"yanzhuguo/linguistic-diversity","repo_kind":"official","path":"semantic_diversity.py","file_url":"https://github.com/yanzhuguo/linguistic-diversity/blob/HEAD/semantic_diversity.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"CC0-1.0","inline_ok":true,"mcp_get_code":{"code_sha256":"225f4d9f0906b5a9"}},{"code_sha256_prefix":"45e046380596b5f1","entry":"ttr","repo":"yanzhuguo/linguistic-diversity","repo_kind":"official","path":"lexical_diversity.py","file_url":"https://github.com/yanzhuguo/linguistic-diversity/blob/HEAD/lexical_diversity.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"CC0-1.0","inline_ok":true,"mcp_get_code":{"code_sha256":"45e046380596b5f1"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}