{"url":"/sota/language-modelling-on-lambada","task":{"name":"Language Modelling","url":"/task/language-modelling","note":null},"dataset":{"name":"LAMBADA","url":"/dataset/lambada"},"category":"Natural Language Processing","categories":["Medical","Miscellaneous","Natural Language Processing"],"category_note":null,"description":"A language model is a model of natural language. Language models are useful for a variety of tasks, including speech recognition, machine translation, natural language generation (generating more human-like text), optical character recognition, route optimization, handwriting recognition, grammar induction, and information retrieval.\r\n\r\nLarge language models (LLMs), currently their most advanced form, are predominantly based on transformers trained on larger datasets (frequently using words scraped from the public internet). They have superseded recurrent neural network-based models, which had previously superseded the purely statistical models, such as word n-gram language model. \r\n\r\nSource: [Wikipedia](https://en.wikipedia.org/wiki/Language_model)","description_from":"task","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","rank":"the archive's row order at snapshot; not re-ranked","rows_end_at":"2025-07-28","rows_withheld_as_spam":0,"metric_values":"the archive's strings, untouched"},"metrics":["Accuracy","Perplexity"],"metric_direction":{"note":"inferred from the metric name only (the archive records no direction); null = not inferred, chart draws points only","by_metric":{"Accuracy":"higher","Perplexity":"lower"}},"counts":{"rows":37,"rows_with_code":30,"rows_with_paper_page":35,"rows_dated":35,"rows_using_additional_data":0},"rows":[{"rank_in_archive_order":1,"model":"PaLM-540B (Few-Shot)","metrics":{"Accuracy":"89.7"},"uses_additional_data":false,"paper_date":"2022-04-05","paper":"/paper/palm-scaling-language-modeling-with-pathways-1","paper_url":"https://arxiv.org/abs/2204.02311v5","paper_title":"PaLM: Scaling Language Modeling with Pathways","code":"https://github.com/lucidrains/CoCa-pytorch","n_code_links":7,"syntology":{"n_ran":30,"n_unverified":7,"n_samples":37,"n_pointer_only_licence":0}},{"rank_in_archive_order":2,"model":"PaLM 2-L (one-shot)","metrics":{"Accuracy":"86.9"},"uses_additional_data":false,"paper_date":"2023-05-17","paper":"/paper/palm-2-technical-report-1","paper_url":"https://arxiv.org/abs/2305.10403v3","paper_title":"PaLM 2 Technical Report","code":"https://github.com/eternityyw/tram-benchmark","n_code_links":1,"syntology":null},{"rank_in_archive_order":3,"model":"GPT-3 175B (Few-Shot)","metrics":{"Accuracy":"86.4","Perplexity":"1.92"},"uses_additional_data":false,"paper_date":"2020-05-28","paper":"/paper/language-models-are-few-shot-learners","paper_url":"https://arxiv.org/abs/2005.14165v4","paper_title":"Language Models are Few-Shot Learners","code":"https://github.com/ggml-org/llama.cpp","n_code_links":67,"syntology":{"n_ran":15,"n_unverified":50,"n_samples":65,"n_pointer_only_licence":4}},{"rank_in_archive_order":4,"model":"LLaMA-65B+CFG (Zero-Shot)","metrics":{"Accuracy":"84.0"},"uses_additional_data":false,"paper_date":"2023-06-30","paper":"/paper/stay-on-topic-with-classifier-free-guidance","paper_url":"https://arxiv.org/abs/2306.17806v1","paper_title":"Stay on topic with Classifier-Free Guidance","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":5,"model":"LLaMA-30B+CFG (zero-shot)","metrics":{"Accuracy":"83.9"},"uses_additional_data":false,"paper_date":"2023-06-30","paper":"/paper/stay-on-topic-with-classifier-free-guidance","paper_url":"https://arxiv.org/abs/2306.17806v1","paper_title":"Stay on topic with Classifier-Free Guidance","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":6,"model":"PaLM 2-M (one-shot)","metrics":{"Accuracy":"83.7"},"uses_additional_data":false,"paper_date":"2023-05-17","paper":"/paper/palm-2-technical-report-1","paper_url":"https://arxiv.org/abs/2305.10403v3","paper_title":"PaLM 2 Technical Report","code":"https://github.com/eternityyw/tram-benchmark","n_code_links":1,"syntology":null},{"rank_in_archive_order":7,"model":"Cohere  Large","metrics":{"Accuracy":"82.33"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":8,"model":"LLaMA-13B+CFG (zero-shot)","metrics":{"Accuracy":"82.2"},"uses_additional_data":false,"paper_date":"2023-06-30","paper":"/paper/stay-on-topic-with-classifier-free-guidance","paper_url":"https://arxiv.org/abs/2306.17806v1","paper_title":"Stay on topic with Classifier-Free Guidance","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":9,"model":"PaLM-540B (One-Shot)","metrics":{"Accuracy":"81.8"},"uses_additional_data":false,"paper_date":"2022-04-05","paper":"/paper/palm-scaling-language-modeling-with-pathways-1","paper_url":"https://arxiv.org/abs/2204.02311v5","paper_title":"PaLM: Scaling Language Modeling with Pathways","code":"https://github.com/lucidrains/CoCa-pytorch","n_code_links":7,"syntology":{"n_ran":30,"n_unverified":7,"n_samples":37,"n_pointer_only_licence":0}},{"rank_in_archive_order":10,"model":"GLaM 62B/64E (One-Shot)","metrics":{"Accuracy":"80.9"},"uses_additional_data":false,"paper_date":"2021-12-13","paper":"/paper/glam-efficient-scaling-of-language-models","paper_url":"https://arxiv.org/abs/2112.06905v2","paper_title":"GLaM: Efficient Scaling of Language Models with Mixture-of-Experts","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":11,"model":"PaLM 2-S (one-shot)","metrics":{"Accuracy":"80.7"},"uses_additional_data":false,"paper_date":"2023-05-17","paper":"/paper/palm-2-technical-report-1","paper_url":"https://arxiv.org/abs/2305.10403v3","paper_title":"PaLM 2 Technical Report","code":"https://github.com/eternityyw/tram-benchmark","n_code_links":1,"syntology":null},{"rank_in_archive_order":12,"model":"GLM-130B (bidirectional attention)","metrics":{"Accuracy":"80.2"},"uses_additional_data":false,"paper_date":"2022-10-05","paper":"/paper/glm-130b-an-open-bilingual-pre-trained-model","paper_url":"https://arxiv.org/abs/2210.02414v2","paper_title":"GLM-130B: An Open Bilingual Pre-trained Model","code":"https://github.com/thudm/chatglm2-6b","n_code_links":9,"syntology":{"n_ran":5,"n_unverified":16,"n_samples":21,"n_pointer_only_licence":0}},{"rank_in_archive_order":13,"model":"SparseGPT (175B, 2:4 Sparsity)","metrics":{"Accuracy":"79.47"},"uses_additional_data":false,"paper_date":"2023-01-02","paper":"/paper/massive-language-models-can-be-accurately","paper_url":"https://arxiv.org/abs/2301.00774v3","paper_title":"SparseGPT: Massive Language Models Can Be Accurately Pruned in One-Shot","code":"https://github.com/nvidia/tensorrt-model-optimizer","n_code_links":6,"syntology":{"n_ran":2,"n_unverified":10,"n_samples":12,"n_pointer_only_licence":9}},{"rank_in_archive_order":14,"model":"SparseGPT (175B, 4:8 Sparsity)","metrics":{"Accuracy":"78.77"},"uses_additional_data":false,"paper_date":"2023-01-02","paper":"/paper/massive-language-models-can-be-accurately","paper_url":"https://arxiv.org/abs/2301.00774v3","paper_title":"SparseGPT: Massive Language Models Can Be Accurately Pruned in One-Shot","code":"https://github.com/nvidia/tensorrt-model-optimizer","n_code_links":6,"syntology":{"n_ran":2,"n_unverified":10,"n_samples":12,"n_pointer_only_licence":9}},{"rank_in_archive_order":15,"model":"PaLM-540B (Zero-Shot)","metrics":{"Accuracy":"77.9"},"uses_additional_data":false,"paper_date":"2022-04-05","paper":"/paper/palm-scaling-language-modeling-with-pathways-1","paper_url":"https://arxiv.org/abs/2204.02311v5","paper_title":"PaLM: Scaling Language Modeling with Pathways","code":"https://github.com/lucidrains/CoCa-pytorch","n_code_links":7,"syntology":{"n_ran":30,"n_unverified":7,"n_samples":37,"n_pointer_only_licence":0}},{"rank_in_archive_order":16,"model":"Chinchilla (Zero-Shot)","metrics":{"Accuracy":"77.7"},"uses_additional_data":false,"paper_date":"2022-03-29","paper":"/paper/training-compute-optimal-large-language","paper_url":"https://arxiv.org/abs/2203.15556v1","paper_title":"Training Compute-Optimal Large Language Models","code":"https://github.com/karpathy/llama2.c","n_code_links":2,"syntology":{"n_ran":8,"n_unverified":3,"n_samples":11,"n_pointer_only_licence":4}},{"rank_in_archive_order":17,"model":"SparseGPT (175B, 50% Sparsity)","metrics":{"Accuracy":"76.51"},"uses_additional_data":false,"paper_date":"2023-01-02","paper":"/paper/massive-language-models-can-be-accurately","paper_url":"https://arxiv.org/abs/2301.00774v3","paper_title":"SparseGPT: Massive Language Models Can Be Accurately Pruned in One-Shot","code":"https://github.com/nvidia/tensorrt-model-optimizer","n_code_links":6,"syntology":{"n_ran":2,"n_unverified":10,"n_samples":12,"n_pointer_only_licence":9}},{"rank_in_archive_order":18,"model":"GPT-3 175B (Zero-Shot)","metrics":{"Accuracy":"76.2","Perplexity":"3.00"},"uses_additional_data":false,"paper_date":"2020-05-28","paper":"/paper/language-models-are-few-shot-learners","paper_url":"https://arxiv.org/abs/2005.14165v4","paper_title":"Language Models are Few-Shot Learners","code":"https://github.com/ggml-org/llama.cpp","n_code_links":67,"syntology":{"n_ran":15,"n_unverified":50,"n_samples":65,"n_pointer_only_licence":4}},{"rank_in_archive_order":19,"model":"OPT-175B","metrics":{"Accuracy":"75.59"},"uses_additional_data":false,"paper_date":"2023-01-02","paper":"/paper/massive-language-models-can-be-accurately","paper_url":"https://arxiv.org/abs/2301.00774v3","paper_title":"SparseGPT: Massive Language Models Can Be Accurately Pruned in One-Shot","code":"https://github.com/nvidia/tensorrt-model-optimizer","n_code_links":6,"syntology":{"n_ran":2,"n_unverified":10,"n_samples":12,"n_pointer_only_licence":9}},{"rank_in_archive_order":20,"model":"GPT-3 13B (Zero-Shot)","metrics":{"Accuracy":"72.5","Perplexity":"3.56"},"uses_additional_data":false,"paper_date":"2020-05-28","paper":"/paper/language-models-are-few-shot-learners","paper_url":"https://arxiv.org/abs/2005.14165v4","paper_title":"Language Models are Few-Shot Learners","code":"https://github.com/ggml-org/llama.cpp","n_code_links":67,"syntology":{"n_ran":15,"n_unverified":50,"n_samples":65,"n_pointer_only_licence":4}},{"rank_in_archive_order":21,"model":"GLM-XXLarge (bidirectional)","metrics":{"Accuracy":"72.35"},"uses_additional_data":false,"paper_date":"2021-03-18","paper":"/paper/all-nlp-tasks-are-generation-tasks-a-general","paper_url":"https://arxiv.org/abs/2103.10360v2","paper_title":"GLM: General Language Model Pretraining with Autoregressive Blank Infilling","code":"https://github.com/thudm/chatglm2-6b","n_code_links":8,"syntology":{"n_ran":1,"n_unverified":0,"n_samples":1,"n_pointer_only_licence":0}},{"rank_in_archive_order":22,"model":"Pythia 12B (0-shot)","metrics":{"Accuracy":"70.46"},"uses_additional_data":false,"paper_date":"2023-04-03","paper":"/paper/pythia-a-suite-for-analyzing-large-language","paper_url":"https://arxiv.org/abs/2304.01373v2","paper_title":"Pythia: A Suite for Analyzing Large Language Models Across Training and Scaling","code":"https://github.com/Lightning-AI/lit-gpt","n_code_links":4,"syntology":null},{"rank_in_archive_order":23,"model":"GPT-3 6.7B (Zero-Shot)","metrics":{"Accuracy":"70.3","Perplexity":"4.00"},"uses_additional_data":false,"paper_date":"2020-05-28","paper":"/paper/language-models-are-few-shot-learners","paper_url":"https://arxiv.org/abs/2005.14165v4","paper_title":"Language Models are Few-Shot Learners","code":"https://github.com/ggml-org/llama.cpp","n_code_links":67,"syntology":{"n_ran":15,"n_unverified":50,"n_samples":65,"n_pointer_only_licence":4}},{"rank_in_archive_order":24,"model":"GPT-J-6B","metrics":{"Accuracy":"69.7","Perplexity":"3.99"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":25,"model":"Mamba-2.8B","metrics":{"Accuracy":"69.2","Perplexity":"4.23"},"uses_additional_data":false,"paper_date":"2023-12-01","paper":"/paper/mamba-linear-time-sequence-modeling-with","paper_url":"https://arxiv.org/abs/2312.00752v2","paper_title":"Mamba: Linear-Time Sequence Modeling with Selective State Spaces","code":"https://github.com/state-spaces/mamba","n_code_links":35,"syntology":{"n_ran":18,"n_unverified":44,"n_samples":62,"n_pointer_only_licence":28}},{"rank_in_archive_order":26,"model":"Pythia 6.9B (0-shot)","metrics":{"Accuracy":"67.28"},"uses_additional_data":false,"paper_date":"2023-04-03","paper":"/paper/pythia-a-suite-for-analyzing-large-language","paper_url":"https://arxiv.org/abs/2304.01373v2","paper_title":"Pythia: A Suite for Analyzing Large Language Models Across Training and Scaling","code":"https://github.com/Lightning-AI/lit-gpt","n_code_links":4,"syntology":null},{"rank_in_archive_order":27,"model":"GLM-XXLarge (unidirectional)","metrics":{"Accuracy":"67.18"},"uses_additional_data":false,"paper_date":"2021-03-18","paper":"/paper/all-nlp-tasks-are-generation-tasks-a-general","paper_url":"https://arxiv.org/abs/2103.10360v2","paper_title":"GLM: General Language Model Pretraining with Autoregressive Blank Infilling","code":"https://github.com/thudm/chatglm2-6b","n_code_links":8,"syntology":{"n_ran":1,"n_unverified":0,"n_samples":1,"n_pointer_only_licence":0}},{"rank_in_archive_order":28,"model":"GPT-3 2.7B (Zero-Shot)","metrics":{"Accuracy":"67.1","Perplexity":"4.60"},"uses_additional_data":false,"paper_date":"2020-05-28","paper":"/paper/language-models-are-few-shot-learners","paper_url":"https://arxiv.org/abs/2005.14165v4","paper_title":"Language Models are Few-Shot Learners","code":"https://github.com/ggml-org/llama.cpp","n_code_links":67,"syntology":{"n_ran":15,"n_unverified":50,"n_samples":65,"n_pointer_only_licence":4}},{"rank_in_archive_order":29,"model":"GPT-2 1.5B (Zero Shot)","metrics":{"Accuracy":"63.24","Perplexity":"8.63"},"uses_additional_data":false,"paper_date":"2019-02-14","paper":"/paper/language-models-are-unsupervised-multitask","paper_url":"https://d4mucfpksywv.cloudfront.net/better-language-models/language-models.pdf","paper_title":"Language Models are Unsupervised Multitask Learners","code":"https://github.com/huggingface/transformers","n_code_links":21,"syntology":null},{"rank_in_archive_order":30,"model":"Universal Transformer (w/ dynamic halting)","metrics":{"Accuracy":"56.25"},"uses_additional_data":false,"paper_date":"2018-07-10","paper":"/paper/universal-transformers","paper_url":"http://arxiv.org/abs/1807.03819v3","paper_title":"Universal Transformers","code":"https://github.com/tensorflow/tensor2tensor","n_code_links":8,"syntology":{"n_ran":14,"n_unverified":11,"n_samples":25,"n_pointer_only_licence":24}},{"rank_in_archive_order":31,"model":"Residual Shuffle-Exchange network","metrics":{"Accuracy":"54.34"},"uses_additional_data":false,"paper_date":"2020-04-06","paper":"/paper/residual-shuffle-exchange-networks-for-fast","paper_url":"https://arxiv.org/abs/2004.04662v4","paper_title":"Residual Shuffle-Exchange Networks for Fast Processing of Long Sequences","code":"https://github.com/LUMII-Syslab/RSE","n_code_links":2,"syntology":null},{"rank_in_archive_order":32,"model":"Gated-Attention Reader (+ features)","metrics":{"Accuracy":"49.0"},"uses_additional_data":false,"paper_date":"2016-10-26","paper":"/paper/broad-context-language-modeling-as-reading","paper_url":"http://arxiv.org/abs/1610.08431v3","paper_title":"Broad Context Language Modeling as Reading Comprehension","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":33,"model":"OPT-175B (50% Sparsity)","metrics":{"Accuracy":"0.02"},"uses_additional_data":false,"paper_date":"2023-01-02","paper":"/paper/massive-language-models-can-be-accurately","paper_url":"https://arxiv.org/abs/2301.00774v3","paper_title":"SparseGPT: Massive Language Models Can Be Accurately Pruned in One-Shot","code":"https://github.com/nvidia/tensorrt-model-optimizer","n_code_links":6,"syntology":{"n_ran":2,"n_unverified":10,"n_samples":12,"n_pointer_only_licence":9}},{"rank_in_archive_order":34,"model":"test","metrics":{"Accuracy":"0.01"},"uses_additional_data":false,"paper_date":"2019-09-29","paper":"/paper/test-time-training-for-out-of-distribution-1","paper_url":"https://arxiv.org/abs/1909.13231v3","paper_title":"Test-Time Training with Self-Supervision for Generalization under Distribution Shifts","code":"https://github.com/yueatsprograms/ttt_cifar_release","n_code_links":3,"syntology":null},{"rank_in_archive_order":35,"model":"Pythia 12B(Zero-Shot)","metrics":{"Perplexity":"3.92"},"uses_additional_data":false,"paper_date":"2023-04-03","paper":"/paper/pythia-a-suite-for-analyzing-large-language","paper_url":"https://arxiv.org/abs/2304.01373v2","paper_title":"Pythia: A Suite for Analyzing Large Language Models Across Training and Scaling","code":"https://github.com/Lightning-AI/lit-gpt","n_code_links":4,"syntology":null},{"rank_in_archive_order":36,"model":"Pythia 6.9B(Zero-Shot)","metrics":{"Perplexity":"4.45"},"uses_additional_data":false,"paper_date":"2023-04-03","paper":"/paper/pythia-a-suite-for-analyzing-large-language","paper_url":"https://arxiv.org/abs/2304.01373v2","paper_title":"Pythia: A Suite for Analyzing Large Language Models Across Training and Scaling","code":"https://github.com/Lightning-AI/lit-gpt","n_code_links":4,"syntology":null},{"rank_in_archive_order":37,"model":"Megatron-Turing NLG 530B (Few-Shot)","metrics":{"Accuracy":"Megatron-Turing NLG 530B (Few-Shot)"},"uses_additional_data":false,"paper_date":"2022-01-28","paper":"/paper/using-deepspeed-and-megatron-to-train","paper_url":"https://arxiv.org/abs/2201.11990v3","paper_title":"Using DeepSpeed and Megatron to Train Megatron-Turing NLG 530B, A Large-Scale Generative Language Model","code":"https://github.com/microsoft/DeepSpeed","n_code_links":2,"syntology":null}],"since_archive":{"claim":"Results that newer papers report for their own method, placed here by Syntology. A model pointed at the cell in the paper's own table; the number was read from that cell and checked against this leaderboard's metric, dataset, split and scale; an independent check that saw this leaderboard's other rows and every other leaderboard on the same dataset accepted it. Not reviewed by the paper's authors or by the archive's editors, and not ranked against the archive rows.","extraction_file_present":true,"measurement":{"test_papers":883,"papers_with_output":881,"judged_true":108,"judged":110,"wilson95_lower":0.9361,"measured_on":"2026-09-24","frozen_commit":"0e3de0df94"},"measurement_note":"blind adjudication of accepted entries on a held-out split of archive papers, rules frozen before the test","coverage":{"sentence":"Syntology has checked 6,264 of the 9,581 papers on this site that are newer than the archive; results from the others appear after they are checked.","complete":false,"papers_newer_than_archive":9581,"papers_checked":6264,"papers_extracted_not_yet_verified":0,"boards_without_verdict":2,"papers_not_yet_extracted":3316},"order":"newest first by month (arXiv date, else the arXiv-id month), then arXiv id descending","columns":["Accuracy"],"entries":[{"paper":"/paper/arxiv-2606-26587","paper_title":"SharQ: Bridging Activation Sparsity and FP4 Quantization for LLM Inference","arxiv_id":"2606.26587","model":"Llama-3.1-8B SharQ","row_label":"Llama-3.1-8B SharQ","configuration":null,"configuration_in_model":false,"configuration_not_shown_because":null,"values":{"Accuracy":"74.67"},"date":"2026-06-25","month_from_arxiv_id":null,"where_in_paper":"Table 1, row “Llama-3.1-8B SharQ”","table_label":"1","table_label_from":"the caption's own label","table_position_1based":1,"caption_head":"Table 1: Zero-shot and few-shot evaluation of SharQ across three LLMs. Accuracy…","cells":[{"column":"Accuracy","cell_text":"74.67","column_header":"Lambada","value":74.67,"table_index_0based":0,"row":3,"col":4}],"code":"2 of 3 ran"}]},"syntology":{"read_at":"2026-09-24T18:15:14+00:00","claim":"Per row: N of M harvested code samples from that row's paper executed on a synthesized fixture; the other M-N are unverified. Not a reproduction of the row's number; not a correctness claim. n_pointer_only_licence counts samples the site points at rather than redistributes (a licence axis, independent of ran/unverified).","rows_with_graph_line":19,"rows_with_any_sample_ran":19,"distinct_papers_with_graph_line":8,"distinct_papers_with_any_sample_ran":8,"samples_over_distinct_papers":{"n_ran":93,"n_unverified":141,"n_samples":234,"n_pointer_only_licence":69,"note":"each paper (arXiv id) counted once, however many rows it is behind; this is the page-level figure"},"samples_row_weighted":{"n_ran":222,"n_unverified":395,"n_samples":617,"n_pointer_only_licence":121,"note":"row-weighted: a paper behind several rows is counted once per row; inflated relative to samples_over_distinct_papers by design, kept for readers summing the per-row syntology blocks"}}}