{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/extracting-training-data-from-large-language","title":"Extracting Training Data from Large Language Models","arxiv_id":"2012.07805","date":"2020-12-14","proceeding":null,"authors":["Nicholas Carlini","Florian Tramer","Eric Wallace","Matthew Jagielski","Ariel Herbert-Voss","Katherine Lee","Adam Roberts","Tom Brown","Dawn Song","Ulfar Erlingsson","Alina Oprea","Colin Raffel"],"abstract":"It has become common to publish large (billion parameter) language models that have been trained on private datasets. This paper demonstrates that in such settings, an adversary can perform a training data extraction attack to recover individual training examples by querying the language model. We demonstrate our attack on GPT-2, a language model trained on scrapes of the public Internet, and are able to extract hundreds of verbatim text sequences from the model's training data. These extracted examples include (public) personally identifiable information (names, phone numbers, and email addresses), IRC conversations, code, and 128-bit UUIDs. Our attack is possible even though each of the above sequences are included in just one document in the training data. We comprehensively evaluate our extraction attack to understand the factors that contribute to its success. Worryingly, we find that larger models are more vulnerable than smaller models. We conclude by drawing lessons and discussing possible safeguards for training large language models.","url_abs":"https://arxiv.org/abs/2012.07805v2","url_pdf":"https://arxiv.org/pdf/2012.07805v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"extracting-training-data-from-large-language","repo_url":"https://github.com/ftramer/LM_Memorization","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"extracting-training-data-from-large-language","repo_url":"https://github.com/shreyansh26/Extracting-Training-Data-from-Large-Langauge-Models","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"extracting-training-data-from-large-language","repo_url":"https://github.com/yonsei-cysec/Language_Model_Memorization","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"unanswered"}}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"cosine-annealing","method_name":"Cosine Annealing"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"discriminative-fine-tuning","method_name":"Discriminative Fine-Tuning"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt-2","method_name":"GPT-2"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-cosine-annealing","method_name":"Linear Warmup With Cosine Annealing"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2012.07805","atlas_url":"https://app.syntology.ai/?focus=2012.07805","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2012.07805"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/shreyansh26/Extracting-Training-Data-from-Large-Langauge-Models","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yonsei-cysec/Language_Model_Memorization","reach":{"status":"unanswered"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ftramer/LM_Memorization","reach":null}],"summary":{"ran_honours":1,"ran_draft_wrong":2,"unverified":1},"by_repo_kind":{"official":{"samples":3,"ran":3,"repositories":1},"listed":{"samples":1,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"f21497e100719c5a","entry":"calculatePerplexity","repo":"ftramer/LM_Memorization","repo_kind":"official","path":"extraction.py","file_url":"https://github.com/ftramer/LM_Memorization/blob/HEAD/extraction.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f21497e100719c5a"}},{"code_sha256_prefix":"2a6129e2c84f477d","entry":"parse_arguments","repo":"ftramer/LM_Memorization","repo_kind":"official","path":"extraction.py","file_url":"https://github.com/ftramer/LM_Memorization/blob/HEAD/extraction.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2a6129e2c84f477d"}},{"code_sha256_prefix":"7a30c79a1d5e467c","entry":"parse_commoncrawl","repo":"ftramer/LM_Memorization","repo_kind":"official","path":"extraction.py","file_url":"https://github.com/ftramer/LM_Memorization/blob/HEAD/extraction.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7a30c79a1d5e467c"}},{"code_sha256_prefix":"c59c1720bffdb7a5","entry":"calculate_perplexity","repo":"shreyansh26/Extracting-Training-Data-from-Large-Langauge-Models","repo_kind":"listed","path":"extraction_top_n.py","file_url":"https://github.com/shreyansh26/Extracting-Training-Data-from-Large-Langauge-Models/blob/HEAD/extraction_top_n.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c59c1720bffdb7a5"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}