{"url":"/dataset/wikitext-103","name":"WikiText-103","full_name":"WikiText-103","description_markdown":"The WikiText language modeling dataset is a collection of over 100 million tokens extracted from the set of verified Good and Featured articles on Wikipedia. The dataset is available under the Creative Commons Attribution-ShareAlike License.\r\n\r\nCompared to the preprocessed version of Penn Treebank (PTB), WikiText-2 is over 2 times larger and WikiText-103 is over 110 times larger. The WikiText dataset also features a far larger vocabulary and retains the original case, punctuation and numbers - all of which are removed in PTB. As it is composed of full articles, the dataset is well suited for models that can take advantage of long term dependencies.\r\n\r\nSource: [The WikiText Long Term Dependency Language Modeling Dataset](https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/)\r\nImage Source: [https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/](https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/)","description_withheld":null,"homepage":"https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/","introduced_date":"2016-09-26","introduced_date_note":null,"introduced_by":{"paper":"/paper/pointer-sentinel-mixture-models","title":"Pointer Sentinel Mixture Models","first_author":"Stephen Merity","url":null},"license":{"name":"CC BY-SA 3.0","url":"https://blog.einstein.ai/the-wikitext-long-term-dependency-language-modeling-dataset/"},"modalities":[{"name":"Texts","url":"/datasets/modality/texts"}],"tasks":[{"name":"Text Generation","url":"/task/text-generation","datasets_with_task":"/datasets/task/text-generation"},{"name":"Language Modelling","url":"/task/language-modelling","datasets_with_task":"/datasets/task/language-modelling"}],"languages":[{"name":"English","url":"/datasets/language/english"}],"variants":["WikiText-103"],"data_loaders":[{"repo":"https://github.com/pytorch/text","url":"https://pytorch.org/text/stable/datasets.html#torchtext.datasets.WikiText103","frameworks":["pytorch"]}],"num_papers_in_archive":560,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28"},"benchmarks":[{"leaderboard":"/sota/language-modelling-on-wikitext-103","task":"Language Modelling","dataset_variant":"WikiText-103","rows":89,"metrics":["Test perplexity","Validation perplexity","Number of params"],"first_row_in_archive_order":{"model":"RETRO (7.5B)","paper":"/paper/improving-language-models-by-retrieving-from","metrics":{"Number of params":"7532M","Test perplexity":"2.4"},"code_links":[{"title":"labmlai/annotated_deep_learning_paper_implementations","url":"https://github.com/labmlai/annotated_deep_learning_paper_implementations"},{"title":"lucidrains/RETRO-pytorch","url":"https://github.com/lucidrains/RETRO-pytorch"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/text-generation-on-wikitext-103","task":"Text Generation","dataset_variant":"WikiText-103","rows":0,"metrics":["Perplexity"],"first_row_in_archive_order":null,"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"}],"papers_with_a_benchmark_row":[{"paper":"/paper/advancing-state-of-the-art-in-language","title":"Advancing State of the Art in Language Modeling","date":"2023-11-28","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/memory-efficient-stochastic-methods-for","title":"Memory-efficient Stochastic methods for Memory-based Transformers","date":"2023-11-14","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/gateloop-fully-data-controlled-linear","title":"GateLoop: Fully Data-Controlled Linear Recurrence for Sequence Modeling","date":"2023-11-03","rows_on_this_dataset":1,"code_links":3,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":5,"samples_ran":3,"samples_unverified":2,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/the-information-pathways-hypothesis","title":"The Information Pathways Hypothesis: Transformers are Dynamic Self-Ensembles","date":"2023-06-02","rows_on_this_dataset":2,"code_links":1,"syntology":null},{"paper":"/paper/primal-attention-self-attention-through","title":"Primal-Attention: Self-attention through Asymmetric Kernel SVD in Primal Representation","date":"2023-05-31","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":1,"samples_ran":1,"samples_unverified":0,"pointer_only_for_licence":1,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/hyena-hierarchy-towards-larger-convolutional","title":"Hyena Hierarchy: Towards Larger Convolutional Language Models","date":"2023-02-21","rows_on_this_dataset":2,"code_links":7,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":5,"samples_ran":5,"samples_unverified":0,"pointer_only_for_licence":4,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/hungry-hungry-hippos-towards-language","title":"Hungry Hungry Hippos: Towards Language Modeling with State Space Models","date":"2022-12-28","rows_on_this_dataset":5,"code_links":3,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":15,"samples_ran":7,"samples_unverified":8,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/you-can-t-pick-your-neighbors-or-can-you-when","title":"You can't pick your neighbors, or can you? When and how to rely on retrieval in the $k$NN-LM","date":"2022-10-28","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":3,"samples_ran":0,"samples_unverified":3,"pointer_only_for_licence":3,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/mega-moving-average-equipped-gated-attention","title":"Mega: Moving Average Equipped Gated Attention","date":"2022-09-21","rows_on_this_dataset":1,"code_links":7,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":13,"samples_ran":12,"samples_unverified":1,"pointer_only_for_licence":8,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/general-purpose-long-context-autoregressive","title":"General-purpose, long-context autoregressive modeling with Perceiver AR","date":"2022-02-15","rows_on_this_dataset":1,"code_links":3,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":12,"samples_ran":6,"samples_unverified":6,"pointer_only_for_licence":1,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/improving-language-models-by-retrieving-from","title":"Improving language models by retrieving from trillions of tokens","date":"2021-12-08","rows_on_this_dataset":1,"code_links":2,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":23,"samples_ran":16,"samples_unverified":7,"pointer_only_for_licence":3,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/efficiently-modeling-long-sequences-with-1","title":"Efficiently Modeling Long Sequences with Structured State Spaces","date":"2021-10-31","rows_on_this_dataset":1,"code_links":8,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":55,"samples_ran":28,"samples_unverified":27,"pointer_only_for_licence":3,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/infty-former-infinite-memory-transformer","title":"$\\infty$-former: Infinite Memory Transformer","date":"2021-09-01","rows_on_this_dataset":7,"code_links":1,"syntology":null},{"paper":"/paper/fnetar-mixing-tokens-with-autoregressive","title":"FNetAR: Mixing Tokens with Autoregressive Fourier Transforms","date":"2021-07-22","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/differentiable-model-compression-via-pseudo","title":"Differentiable Model Compression via Pseudo Quantization Noise","date":"2021-04-20","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/revisiting-simple-neural-probabilistic","title":"Revisiting Simple Neural Probabilistic Language Models","date":"2021-04-08","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":3,"samples_ran":0,"samples_unverified":3,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/finetuning-pretrained-transformers-into-rnns","title":"Finetuning Pretrained Transformers into RNNs","date":"2021-03-24","rows_on_this_dataset":1,"code_links":2,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":8,"samples_ran":3,"samples_unverified":5,"pointer_only_for_licence":4,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/all-nlp-tasks-are-generation-tasks-a-general","title":"GLM: General Language Model Pretraining with Autoregressive Blank Infilling","date":"2021-03-18","rows_on_this_dataset":2,"code_links":8,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":1,"samples_ran":1,"samples_unverified":0,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/random-feature-attention-1","title":"Random Feature Attention","date":"2021-03-03","rows_on_this_dataset":2,"code_links":0,"syntology":null},{"paper":"/paper/when-attention-meets-fast-recurrence-training","title":"When Attention Meets Fast Recurrence: Training Language Models with Reduced Compute","date":"2021-02-24","rows_on_this_dataset":2,"code_links":1,"syntology":null},{"paper":"/paper/subformer-a-parameter-reduced-transformer","title":"Subformer: A Parameter Reduced Transformer","date":"2021-01-01","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/shortformer-better-language-modeling-using","title":"Shortformer: Better Language Modeling using Shorter Inputs","date":"2020-12-31","rows_on_this_dataset":2,"code_links":1,"syntology":null},{"paper":"/paper/rethinking-attention-with-performers","title":"Rethinking Attention with Performers","date":"2020-09-30","rows_on_this_dataset":1,"code_links":7,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":16,"samples_ran":9,"samples_unverified":7,"pointer_only_for_licence":6,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/pay-attention-when-required","title":"Pay Attention when Required","date":"2020-09-09","rows_on_this_dataset":2,"code_links":2,"syntology":null},{"paper":"/paper/delight-very-deep-and-light-weight","title":"DeLighT: Deep and Light-weight Transformer","date":"2020-08-03","rows_on_this_dataset":1,"code_links":2,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":3,"samples_ran":0,"samples_unverified":3,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/transformers-are-rnns-fast-autoregressive","title":"Transformers are RNNs: Fast Autoregressive Transformers with Linear Attention","date":"2020-06-29","rows_on_this_dataset":1,"code_links":8,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":8,"samples_ran":3,"samples_unverified":5,"pointer_only_for_licence":3,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/how-much-complexity-does-an-rnn-architecture","title":"How much complexity does an RNN architecture need to learn syntax-sensitive dependencies?","date":"2020-05-17","rows_on_this_dataset":3,"code_links":1,"syntology":null},{"paper":"/paper/segabert-pre-training-of-segment-aware-bert","title":"Segatron: Segment-Aware Transformer for Language Modeling and Understanding","date":"2020-04-30","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/efficient-content-based-sparse-attention-with-1","title":"Efficient Content-Based Sparse Attention with Routing Transformers","date":"2020-03-12","rows_on_this_dataset":1,"code_links":2,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":3,"samples_ran":3,"samples_unverified":0,"pointer_only_for_licence":2,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/accessing-higher-level-representations-in","title":"Addressing Some Limitations of Transformers with Feedback Memory","date":"2020-02-21","rows_on_this_dataset":2,"code_links":4,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":3,"samples_ran":3,"samples_unverified":0,"pointer_only_for_licence":2,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/time-aware-large-kernel-convolutions","title":"Time-aware Large Kernel Convolutions","date":"2020-02-08","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":7,"samples_ran":5,"samples_unverified":2,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/reformer-the-efficient-transformer-1","title":"Reformer: The Efficient Transformer","date":"2020-01-13","rows_on_this_dataset":1,"code_links":10,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":8,"samples_ran":6,"samples_unverified":2,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/compressive-transformers-for-long-range-1","title":"Compressive Transformers for Long-Range Sequence Modelling","date":"2019-11-13","rows_on_this_dataset":1,"code_links":6,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":11,"samples_ran":3,"samples_unverified":8,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/improving-transformer-models-by-reordering","title":"Improving Transformer Models by Reordering their Sublayers","date":"2019-11-10","rows_on_this_dataset":1,"code_links":2,"syntology":null},{"paper":"/paper/generalization-through-memorization-nearest","title":"Generalization through Memorization: Nearest Neighbor Language Models","date":"2019-11-01","rows_on_this_dataset":2,"code_links":5,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":3,"samples_ran":3,"samples_unverified":0,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/on-the-adequacy-of-untuned-warmup-for","title":"On the adequacy of untuned warmup for adaptive optimization","date":"2019-10-09","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":4,"samples_ran":0,"samples_unverified":4,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/alleviating-sequence-information-loss-with","title":"Alleviating Sequence Information Loss with Data Overlapping and Prime Batch Sizes","date":"2019-09-18","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/megatron-lm-training-multi-billion-parameter","title":"Megatron-LM: Training Multi-Billion Parameter Language Models Using Model Parallelism","date":"2019-09-17","rows_on_this_dataset":1,"code_links":10,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":47,"samples_ran":12,"samples_unverified":35,"pointer_only_for_licence":15,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/deep-equilibrium-models","title":"Deep Equilibrium Models","date":"2019-09-03","rows_on_this_dataset":3,"code_links":11,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":12,"samples_ran":3,"samples_unverified":9,"pointer_only_for_licence":4,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/augmenting-self-attention-with-persistent","title":"Augmenting Self-attention with Persistent Memory","date":"2019-07-02","rows_on_this_dataset":1,"code_links":2,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":5,"samples_ran":5,"samples_unverified":0,"pointer_only_for_licence":4,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/improving-neural-language-modeling-via","title":"Improving Neural Language Modeling via Adversarial Training","date":"2019-06-10","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/improving-neural-language-models-by","title":"Improving Neural Language Models by Segmenting, Attending, and Predicting the Future","date":"2019-06-04","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/190409408","title":"Language Models with Transformers","date":"2019-04-20","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":2,"samples_ran":2,"samples_unverified":0,"pointer_only_for_licence":2,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/dynamic-evaluation-of-transformer-language","title":"Dynamic Evaluation of Transformer Language Models","date":"2019-04-17","rows_on_this_dataset":2,"code_links":1,"syntology":null},{"paper":"/paper/language-models-are-unsupervised-multitask","title":"Language Models are Unsupervised Multitask Learners","date":"2019-02-14","rows_on_this_dataset":4,"code_links":21,"syntology":null},{"paper":"/paper/transformer-xl-attentive-language-models","title":"Transformer-XL: Attentive Language Models Beyond a Fixed-Length Context","date":"2019-01-09","rows_on_this_dataset":2,"code_links":37,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":143,"samples_ran":63,"samples_unverified":80,"pointer_only_for_licence":43,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/trellis-networks-for-sequence-modeling","title":"Trellis Networks for Sequence Modeling","date":"2018-10-15","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":8,"samples_ran":2,"samples_unverified":6,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/adaptive-input-representations-for-neural","title":"Adaptive Input Representations for Neural Language Modeling","date":"2018-09-28","rows_on_this_dataset":1,"code_links":3,"syntology":null},{"paper":"/paper/relational-recurrent-neural-networks","title":"Relational recurrent neural networks","date":"2018-06-05","rows_on_this_dataset":1,"code_links":2,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":6,"samples_ran":1,"samples_unverified":5,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/fast-parametric-learning-with-activation","title":"Fast Parametric Learning with Activation Memorization","date":"2018-03-27","rows_on_this_dataset":4,"code_links":0,"syntology":null},{"paper":"/paper/an-analysis-of-neural-language-modeling-at","title":"An Analysis of Neural Language Modeling at Multiple Scales","date":"2018-03-22","rows_on_this_dataset":1,"code_links":12,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":18,"samples_ran":3,"samples_unverified":15,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/an-empirical-evaluation-of-generic","title":"An Empirical Evaluation of Generic Convolutional and Recurrent Networks for Sequence Modeling","date":"2018-03-04","rows_on_this_dataset":1,"code_links":35,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":10,"samples_ran":2,"samples_unverified":8,"pointer_only_for_licence":1,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/convolutional-sequence-modeling-revisited","title":"Convolutional Sequence Modeling Revisited","date":"2018-01-01","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/language-modeling-with-gated-convolutional","title":"Language Modeling with Gated Convolutional Networks","date":"2016-12-23","rows_on_this_dataset":2,"code_links":11,"syntology":null},{"paper":"/paper/improving-neural-language-models-with-a","title":"Improving Neural Language Models with a Continuous Cache","date":"2016-12-13","rows_on_this_dataset":3,"code_links":14,"syntology":null}],"syntology_totals":{"read_at":"2026-09-24T18:15:14+00:00","papers_with_samples":31,"samples_harvested":461,"samples_ran":210,"samples_unverified":251,"pointer_only_for_licence":109,"papers_with_no_sample_that_ran":4,"note":"the per-paper counts above, summed; not a rate"},"papers_note":"The archive never published its papers-using-dataset list; these are papers with a leaderboard row on this dataset's benchmarks."}