{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/realtoxicityprompts-evaluating-neural-toxic","title":"RealToxicityPrompts: Evaluating Neural Toxic Degeneration in Language Models","arxiv_id":"2009.11462","date":"2020-09-24","proceeding":"Findings of the Association for Computational Linguistics 2020","authors":["Samuel Gehman","Suchin Gururangan","Maarten Sap","Yejin Choi","Noah A. Smith"],"abstract":"Pretrained neural language models (LMs) are prone to generating racist, sexist, or otherwise toxic language which hinders their safe deployment. We investigate the extent to which pretrained LMs can be prompted to generate toxic language, and the effectiveness of controllable text generation algorithms at preventing such toxic degeneration. We create and release RealToxicityPrompts, a dataset of 100K naturally occurring, sentence-level prompts derived from a large corpus of English web text, paired with toxicity scores from a widely-used toxicity classifier. Using RealToxicityPrompts, we find that pretrained LMs can degenerate into toxic text even from seemingly innocuous prompts. We empirically assess several controllable generation methods, and find that while data- or compute-intensive methods (e.g., adaptive pretraining on non-toxic data) are more effective at steering away from toxicity than simpler solutions (e.g., banning \"bad\" words), no current method is failsafe against neural toxic degeneration. To pinpoint the potential cause of such persistent toxic degeneration, we analyze two web text corpora used to pretrain several LMs (including GPT-2; Radford et. al, 2019), and find a significant amount of offensive, factually unreliable, and otherwise toxic content. Our work provides a test bed for evaluating toxic generations by LMs and stresses the need for better data selection processes for pretraining.","url_abs":"https://arxiv.org/abs/2009.11462v2","url_pdf":"https://arxiv.org/pdf/2009.11462v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"realtoxicityprompts-evaluating-neural-toxic","repo_url":"https://github.com/allenai/real-toxicity-prompts","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"realtoxicityprompts-evaluating-neural-toxic","repo_url":"https://github.com/for-ai/llm-profiling-toolkit","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"realtoxicityprompts-evaluating-neural-toxic","repo_url":"https://github.com/tomekkorbak/pretraining-with-human-feedback","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"sentence","task_name":"Sentence"},{"task_slug":"text-generation","task_name":"Text Generation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2009.11462","atlas_url":"https://app.syntology.ai/?focus=2009.11462","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2009.11462"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/for-ai/llm-profiling-toolkit","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/tomekkorbak/pretraining-with-human-feedback","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/allenai/real-toxicity-prompts","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_fixture":2,"unverified":3},"by_repo_kind":{"official":{"samples":5,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"bbebd95a156061b0","entry":"to_var","repo":"allenai/real-toxicity-prompts","repo_kind":"official","path":"generation/pplm_generation.py","file_url":"https://github.com/allenai/real-toxicity-prompts/blob/HEAD/generation/pplm_generation.py","link_basis":"plan_row","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bbebd95a156061b0"}},{"code_sha256_prefix":"ac71b1f4007ef1dd","entry":"top_k_filter","repo":"allenai/real-toxicity-prompts","repo_kind":"official","path":"generation/pplm_generation.py","file_url":"https://github.com/allenai/real-toxicity-prompts/blob/HEAD/generation/pplm_generation.py","link_basis":"plan_row","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ac71b1f4007ef1dd"}},{"code_sha256_prefix":"5f768e9c3c410ca0","entry":"adjust_length_to_model","repo":"allenai/real-toxicity-prompts","repo_kind":"official","path":"generation/gpt2_generation.py","file_url":"https://github.com/allenai/real-toxicity-prompts/blob/HEAD/generation/gpt2_generation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5f768e9c3c410ca0"}},{"code_sha256_prefix":"af7ceb2cad5d5b2c","entry":"first","repo":"allenai/real-toxicity-prompts","repo_kind":"official","path":"utils/utils.py","file_url":"https://github.com/allenai/real-toxicity-prompts/blob/HEAD/utils/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"af7ceb2cad5d5b2c"}},{"code_sha256_prefix":"51f543f4f2fda251","entry":"unpack_scores","repo":"allenai/real-toxicity-prompts","repo_kind":"official","path":"utils/perspective_api.py","file_url":"https://github.com/allenai/real-toxicity-prompts/blob/HEAD/utils/perspective_api.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"51f543f4f2fda251"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}