{"url":"/method/chinchilla","slug":"chinchilla","name":"Chinchilla","full_name":"Chinchilla","full_name_withheld":false,"description_markdown":"Chinchilla is a 70B parameters model trained as a compute-optimal model with 1.4 trillion tokens. Findings suggest that these types of models are trained optimally by equally scaling both model size and training tokens. It uses the same compute budget as Gopher but with 4x more training data. Chinchilla and Gopher are trained for the same number of FLOPs. It is trained using [MassiveText](/dataset/massivetext) using a slightly modified SentencePiece tokenizer. More architectural details in the paper.","description_state":"present","introduced_year":null,"introduced_by":{"title":null,"paper":null,"first_author":null,"n_authors":0,"url_abs":null,"archive_paper_url":null},"source":{"url":"https://arxiv.org/abs/2203.15556v1","title":"Training Compute-Optimal Large Language Models","url_on_a_paper_host":true},"code_snippet_url":null,"code_snippet_url_on_a_code_host":false,"categories":[{"area":"Natural Language Processing","area_id":"natural-language-processing","collection":"Language Models","url":"/methods/category/language-models","pwc_aliases":[]}],"n_papers_tagged":25,"archive_num_papers":null,"papers_newest_first":[{"paper":"/paper/superposition-yields-robust-neural-scaling","title":"Superposition Yields Robust Neural Scaling","date":"2025-05-15","arxiv_id":"2505.10465","n_code_links":1,"syntology":null},{"paper":null,"title":"Compute-Optimal LLMs Provably Generalize Better With Scale","date":"2025-04-21","arxiv_id":"2504.15208","n_code_links":0,"syntology":null},{"paper":null,"title":"Scaling Inference-Efficient Language Models","date":"2025-01-30","arxiv_id":"2501.18107","n_code_links":0,"syntology":null},{"paper":"/paper/physics-of-skill-learning","title":"Physics of Skill Learning","date":"2025-01-21","arxiv_id":"2501.12391","n_code_links":1,"syntology":null},{"paper":null,"title":"The Journey Matters: Average Parameter Count over Pre-training Unifies Sparse and Dense Scaling Laws","date":"2025-01-21","arxiv_id":"2501.12486","n_code_links":0,"syntology":null},{"paper":"/paper/how-much-can-we-forget-about-data","title":"How Much Can We Forget about Data Contamination?","date":"2024-10-04","arxiv_id":"2410.03249","n_code_links":1,"syntology":{"ran":1,"of":2,"unverified":1,"pointer_only":0}},{"paper":null,"title":"Energy Estimation of Last Mile Electric Vehicle Routes","date":"2024-08-21","arxiv_id":"2408.12006","n_code_links":0,"syntology":null},{"paper":null,"title":"Scaling Law with Learning Rate Annealing","date":"2024-08-20","arxiv_id":"2408.11029","n_code_links":0,"syntology":null},{"paper":null,"title":"Time Matters: Scaling Laws for Any Budget","date":"2024-06-27","arxiv_id":"2406.18922","n_code_links":0,"syntology":null},{"paper":"/paper/reconciling-kaplan-and-chinchilla-scaling","title":"Reconciling Kaplan and Chinchilla Scaling Laws","date":"2024-06-12","arxiv_id":"2406.12907","n_code_links":1,"syntology":{"ran":2,"of":2,"unverified":0,"pointer_only":2}},{"paper":"/paper/revisiting-moe-and-dense-speed-accuracy","title":"Revisiting MoE and Dense Speed-Accuracy Comparisons for LLM Training","date":"2024-05-23","arxiv_id":"2405.15052","n_code_links":1,"syntology":null},{"paper":null,"title":"More Compute Is What You Need","date":"2024-04-30","arxiv_id":"2404.19484","n_code_links":0,"syntology":null},{"paper":"/paper/minicpm-unveiling-the-potential-of-small","title":"MiniCPM: Unveiling the Potential of Small Language Models with Scalable Training Strategies","date":"2024-04-09","arxiv_id":"2404.06395","n_code_links":3,"syntology":{"ran":1,"of":1,"unverified":0,"pointer_only":0}},{"paper":null,"title":"VBART: The Turkish LLM","date":"2024-03-02","arxiv_id":"2403.01308","n_code_links":0,"syntology":null},{"paper":null,"title":"A Resource Model For Neural Scaling Law","date":"2024-02-07","arxiv_id":"2402.05164","n_code_links":0,"syntology":null},{"paper":null,"title":"Beyond Chinchilla-Optimal: Accounting for Inference in Language Model Scaling Laws","date":"2023-12-31","arxiv_id":"2401.00448","n_code_links":0,"syntology":null},{"paper":"/paper/the-falcon-series-of-open-language-models","title":"The Falcon Series of Open Language Models","date":"2023-11-28","arxiv_id":"2311.16867","n_code_links":0,"syntology":null},{"paper":"/paper/language-modeling-is-compression","title":"Language Modeling Is Compression","date":"2023-09-19","arxiv_id":"2309.10668","n_code_links":1,"syntology":{"ran":6,"of":8,"unverified":2,"pointer_only":0}},{"paper":null,"title":"Does Circuit Analysis Interpretability Scale? Evidence from Multiple Choice Capabilities in Chinchilla","date":"2023-07-18","arxiv_id":"2307.09458","n_code_links":0,"syntology":null},{"paper":null,"title":"Tune As You Scale: Hyperparameter Optimization For Compute Efficient Training","date":"2023-06-13","arxiv_id":"2306.08055","n_code_links":0,"syntology":null},{"paper":"/paper/cerebras-gpt-open-compute-optimal-language","title":"Cerebras-GPT: Open Compute-Optimal Language Models Trained on the Cerebras Wafer-Scale Cluster","date":"2023-04-06","arxiv_id":"2304.03208","n_code_links":2,"syntology":null},{"paper":"/paper/accelerating-large-language-model-decoding","title":"Accelerating Large Language Model Decoding with Speculative Sampling","date":"2023-02-02","arxiv_id":"2302.01318","n_code_links":5,"syntology":null},{"paper":null,"title":"An Information-Theoretic Analysis of Compute-Optimal Neural Scaling Laws","date":"2022-12-02","arxiv_id":"2212.01365","n_code_links":0,"syntology":null},{"paper":"/paper/galactica-a-large-language-model-for-science-1","title":"Galactica: A Large Language Model for Science","date":"2022-11-16","arxiv_id":"2211.09085","n_code_links":1,"syntology":{"ran":0,"of":2,"unverified":2,"pointer_only":0}},{"paper":"/paper/training-compute-optimal-large-language","title":"Training Compute-Optimal Large Language Models","date":"2022-03-29","arxiv_id":"2203.15556","n_code_links":2,"syntology":{"ran":8,"of":11,"unverified":3,"pointer_only":4}}],"papers_shown":25,"tasks":[{"task":"/task/language-modelling","name":"Language Modelling","papers":8},{"task":"/task/language-modeling","name":"Language Modeling","papers":5},{"task":"/task/large-language-model","name":"Large Language Model","papers":4},{"task":"/task/question-answering","name":"Question Answering","papers":4},{"task":"/task/mmlu","name":"MMLU","papers":3},{"task":"/task/multi-task-language-understanding","name":"Multi-task Language Understanding","papers":3},{"task":"/task/model","name":"model","papers":3},{"task":"/task/anachronisms","name":"Anachronisms","papers":2},{"task":"/task/common-sense-reasoning","name":"Common Sense Reasoning","papers":2},{"task":"/task/decoder","name":"Decoder","papers":2},{"task":"/task/mathematical-reasoning","name":"Mathematical Reasoning","papers":2},{"task":"/task/multiple-choice-qa","name":"Multiple Choice Question Answering (MCQA)","papers":2},{"task":"/task/sentence-completion","name":"Sentence Completion","papers":2},{"task":"/task/word-sense-disambiguation","name":"Word Sense Disambiguation","papers":2},{"task":"/task/abstractive-text-summarization","name":"Abstractive Text Summarization","papers":1},{"task":"/task/analogical-similarity","name":"Analogical Similarity","papers":1},{"task":"/task/analytic-entailment","name":"Analytic Entailment","papers":1},{"task":"/task/bayesian-optimization","name":"Bayesian Optimization","papers":1},{"task":"/task/bias-detection","name":"Bias Detection","papers":1},{"task":"/task/causal-judgment","name":"Causal Judgment","papers":1}],"tasks_shown":20,"n_tasks":106,"usage_by_year":[{"year":"2022","papers":3},{"year":"2023","papers":7},{"year":"2024","papers":10},{"year":"2025","papers":5}],"row_source":"embedded","archive":{"source":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","archive_url":"https://paperswithcode.com/method/chinchilla"},"syntology_read_at":"2026-09-24T18:15:14+00:00"}