{"url":"/dataset/openwebtext","name":"OpenWebText","full_name":null,"description_markdown":"**OpenWebText** is an open-source recreation of the [WebText](/dataset/webtext) corpus. The text is web content extracted from URLs shared on Reddit with at least three upvotes. (38GB).\r\n\r\nSource: [RoBERTa: A Robustly Optimized BERT Pretraining Approach](https://arxiv.org/abs/1907.11692)","description_withheld":null,"homepage":"https://skylion007.github.io/OpenWebTextCorpus/","introduced_date":null,"introduced_date_note":null,"introduced_by":{"paper":null,"title":"OpenWebText corpus","first_author":null,"url":"http://Skylion007.github.io/OpenWebTextCorpus"},"license":{"name":"Custom","url":"https://skylion007.github.io/OpenWebTextCorpus/"},"modalities":[{"name":"Texts","url":"/datasets/modality/texts"}],"tasks":[{"name":"Text Classification","url":"/task/text-classification","datasets_with_task":"/datasets/task/text-classification"},{"name":"Text Generation","url":"/task/text-generation","datasets_with_task":"/datasets/task/text-generation"},{"name":"Language Modelling","url":"/task/language-modelling","datasets_with_task":"/datasets/task/language-modelling"}],"languages":[{"name":"Kabyle","url":"/datasets/language/kabyle"}],"variants":["OpenWebText"],"data_loaders":[{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/Skylion007/openwebtext","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/Bingsu/openwebtext_20p","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/openwebtext","frameworks":["tf","pytorch","jax"]}],"num_papers_in_archive":207,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28"},"benchmarks":[{"leaderboard":"/sota/language-modelling-on-openwebtext","task":"Language Modelling","dataset_variant":"OpenWebText","rows":12,"metrics":["eval_perplexity","eval_loss","parameters"],"first_row_in_archive_order":{"model":"MDLM-Prime","paper":"/paper/beyond-masked-and-unmasked-discrete-diffusion","metrics":{"eval_perplexity":"15.36","parameters":"131M"},"code_links":[]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/text-generation-on-openwebtext","task":"Text Generation","dataset_variant":"OpenWebText","rows":3,"metrics":["eval_loss"],"first_row_in_archive_order":{"model":"GPT2-Hermite","paper":"/paper/learnable-polynomial-trigonometric-and","metrics":{"eval_loss":"2.91"},"code_links":[{"title":"K-H-Ismail/torchortho","url":"https://github.com/K-H-Ismail/torchortho"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"}],"papers_with_a_benchmark_row":[{"paper":"/paper/beyond-masked-and-unmasked-discrete-diffusion","title":"Beyond Masked and Unmasked: Discrete Diffusion Models via Partial Masking","date":"2025-05-24","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/block-diffusion-interpolating-between","title":"Block Diffusion: Interpolating Between Autoregressive and Diffusion Language Models","date":"2025-03-12","rows_on_this_dataset":1,"code_links":2,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":15,"samples_ran":7,"samples_unverified":8,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/learnable-polynomial-trigonometric-and","title":"Polynomial, trigonometric, and tropical activations","date":"2025-02-03","rows_on_this_dataset":5,"code_links":1,"syntology":null},{"paper":"/paper/energy-based-diffusion-language-models-for","title":"Energy-Based Diffusion Language Models for Text Generation","date":"2024-10-28","rows_on_this_dataset":2,"code_links":0,"syntology":null},{"paper":"/paper/loop-residual-neural-networks-for-iterative","title":"Loop Neural Networks for Parameter Sharing","date":"2024-09-21","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/simple-and-effective-masked-diffusion","title":"Simple and Effective Masked Diffusion Language Models","date":"2024-06-11","rows_on_this_dataset":2,"code_links":2,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":1,"samples_ran":1,"samples_unverified":0,"pointer_only_for_licence":1,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/simplified-and-generalized-masked-diffusion","title":"Simplified and Generalized Masked Diffusion for Discrete Data","date":"2024-06-06","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":21,"samples_ran":6,"samples_unverified":15,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/discrete-diffusion-language-modeling-by","title":"Discrete Diffusion Modeling by Estimating the Ratios of the Data Distribution","date":"2023-10-25","rows_on_this_dataset":1,"code_links":4,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":18,"samples_ran":13,"samples_unverified":5,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/language-models-are-unsupervised-multitask","title":"Language Models are Unsupervised Multitask Learners","date":"2019-02-14","rows_on_this_dataset":1,"code_links":21,"syntology":null}],"syntology_totals":{"read_at":"2026-09-24T18:15:14+00:00","papers_with_samples":4,"samples_harvested":55,"samples_ran":27,"samples_unverified":28,"pointer_only_for_licence":1,"papers_with_no_sample_that_ran":0,"note":"the per-paper counts above, summed; not a rate"},"papers_note":"The archive never published its papers-using-dataset list; these are papers with a leaderboard row on this dataset's benchmarks."}