{"url":"/method/opt","slug":"opt","name":"OPT","full_name":"OPT","full_name_withheld":false,"description_markdown":"**OPT** is a suite of decoder-only pre-trained transformers ranging from 125M to 175B parameters. The model uses an AdamW optimizer and weight decay of 0.1. It follows a linear learning rate schedule, warming up from 0 to the maximum learning rate over the first 2000 steps in OPT-175B, or over 375M tokens in the smaller models, and decaying down to 10% of the maximum LR over 300B tokens. The batch sizes range from 0.5M to 4M depending on the model size and is kept constant throughout the course of training.","description_state":"present","introduced_year":null,"introduced_by":{"title":"OPT: Open Pre-trained Transformer Language Models","paper":"/paper/opt-open-pre-trained-transformer-language","first_author":"Susan Zhang","n_authors":19,"url_abs":null,"archive_paper_url":"https://paperswithcode.com/paper/opt-open-pre-trained-transformer-language"},"source":{"url":"https://arxiv.org/abs/2205.01068v4","title":"OPT: Open Pre-trained Transformer Language Models","url_on_a_paper_host":true},"code_snippet_url":null,"code_snippet_url_on_a_code_host":false,"categories":[{"area":"Natural Language Processing","area_id":"natural-language-processing","collection":"Language Models","url":"/methods/category/language-models","pwc_aliases":[]}],"n_papers_tagged":285,"archive_num_papers":285,"papers_newest_first":[{"paper":null,"title":"Incentivizing High-quality Participation From Federated Learning Agents","date":"2025-06-20","arxiv_id":"2506.16731","n_code_links":0,"syntology":null},{"paper":"/paper/attribution-guided-pruning-for-compression","title":"Attribution-guided Pruning for Compression, Circuit Discovery, and Targeted Correction in LLMs","date":"2025-06-16","arxiv_id":"2506.13727","n_code_links":1,"syntology":null},{"paper":null,"title":"Mixture of Weight-shared Heterogeneous Group Attention Experts for Dynamic Token-wise KV Optimization","date":"2025-06-16","arxiv_id":"2506.13541","n_code_links":0,"syntology":null},{"paper":null,"title":"TensorSLM: Energy-efficient Embedding Compression of Sub-billion Parameter Language Models on Low-end Devices","date":"2025-06-16","arxiv_id":"2506.13514","n_code_links":0,"syntology":null},{"paper":null,"title":"FZOO: Fast Zeroth-Order Optimizer for Fine-Tuning Large Language Models towards Adam-Scale Speed","date":"2025-06-10","arxiv_id":"2506.09034","n_code_links":0,"syntology":null},{"paper":null,"title":"Conservative Bias in Large Language Models: Measuring Relation Predictions","date":"2025-06-09","arxiv_id":"2506.08120","n_code_links":0,"syntology":null},{"paper":null,"title":"NQKV: A KV Cache Quantization Scheme Based on Normal Distribution Characteristics","date":"2025-05-22","arxiv_id":"2505.16210","n_code_links":0,"syntology":null},{"paper":"/paper/polar-sparsity-high-throughput-batched-llm","title":"Polar Sparsity: High Throughput Batched LLM Inferencing with Scalable Contextual Sparsity","date":"2025-05-20","arxiv_id":"2505.14884","n_code_links":1,"syntology":{"ran":3,"of":3,"unverified":0,"pointer_only":0}},{"paper":null,"title":"Optimizing Energy Consumption in Stochastic Production Systems: Using a Simulation-Based Approach for Stopping Policy","date":"2025-05-14","arxiv_id":"2505.11536","n_code_links":0,"syntology":null},{"paper":null,"title":"Anticipating Gaming to Incentivize Improvement: Guiding Agents in (Fair) Strategic Classification","date":"2025-05-08","arxiv_id":"2505.05594","n_code_links":0,"syntology":null},{"paper":null,"title":"SPAP: Structured Pruning via Alternating Optimization and Penalty Methods","date":"2025-05-06","arxiv_id":"2505.03373","n_code_links":0,"syntology":null},{"paper":null,"title":"Efficient Shapley Value-based Non-Uniform Pruning of Large Language Models","date":"2025-05-03","arxiv_id":"2505.01731","n_code_links":0,"syntology":null},{"paper":"/paper/unidetox-universal-detoxification-of-large","title":"UniDetox: Universal Detoxification of Large Language Models via Dataset Distillation","date":"2025-04-29","arxiv_id":"2504.20500","n_code_links":1,"syntology":{"ran":1,"of":1,"unverified":0,"pointer_only":0}},{"paper":null,"title":"PIN-WM: Learning Physics-INformed World Models for Non-Prehensile Manipulation","date":"2025-04-23","arxiv_id":"2504.16693","n_code_links":0,"syntology":null},{"paper":null,"title":"Transferable text data distillation by trajectory matching","date":"2025-04-14","arxiv_id":"2504.09818","n_code_links":0,"syntology":null},{"paper":null,"title":"Maximum Redundancy Pruning: A Principle-Driven Layerwise Sparsity Allocation for LLMs","date":"2025-03-24","arxiv_id":"2503.18377","n_code_links":0,"syntology":null},{"paper":null,"title":"MuMA: 3D PBR Texturing via Multi-Channel Multi-View Generation and Agentic Post-Processing","date":"2025-03-24","arxiv_id":"2503.18461","n_code_links":0,"syntology":null},{"paper":null,"title":"How does Bike Absence Influence Mode Shifts Among Dockless Bike-Sharing Users? Evidence From Nanjing, China","date":"2025-03-18","arxiv_id":"2503.14265","n_code_links":0,"syntology":null},{"paper":null,"title":"Large language models in finance : what is financial sentiment?","date":"2025-03-05","arxiv_id":"2503.03612","n_code_links":0,"syntology":null},{"paper":"/paper/probe-pruning-accelerating-llms-through","title":"Probe Pruning: Accelerating LLMs through Dynamic Pruning via Model-Probing","date":"2025-02-21","arxiv_id":"2502.15618","n_code_links":1,"syntology":{"ran":2,"of":3,"unverified":1,"pointer_only":1}},{"paper":"/paper/enhancing-cross-tokenizer-knowledge","title":"Enhancing Cross-Tokenizer Knowledge Distillation with Contextual Dynamical Mapping","date":"2025-02-16","arxiv_id":"2502.11104","n_code_links":1,"syntology":null},{"paper":"/paper/the-odyssey-of-the-fittest-can-agents-survive","title":"The Odyssey of the Fittest: Can Agents Survive and Still Be Good?","date":"2025-02-08","arxiv_id":"2502.05442","n_code_links":1,"syntology":null},{"paper":"/paper/green-code-optimizing-energy-efficiency-in","title":"GREEN-CODE: Learning to Optimize Energy Efficiency in LLM-based Code Generation","date":"2025-01-19","arxiv_id":"2501.11006","n_code_links":1,"syntology":null},{"paper":null,"title":"FASP: Fast and Accurate Structured Pruning of Large Language Models","date":"2025-01-16","arxiv_id":"2501.09412","n_code_links":0,"syntology":null},{"paper":"/paper/multilingual-llms-struggle-to-link","title":"Multilingual LLMs Struggle to Link Orthography and Semantics in Bilingual Word Processing","date":"2025-01-15","arxiv_id":"2501.09127","n_code_links":1,"syntology":null},{"paper":null,"title":"Has an AI model been trained on your images?","date":"2025-01-11","arxiv_id":"2501.06399","n_code_links":0,"syntology":null},{"paper":"/paper/explore-activation-sparsity-in-recurrent-llms","title":"Explore Activation Sparsity in Recurrent LLMs for Energy-Efficient Neuromorphic Computing","date":"2025-01-09","arxiv_id":"2501.16337","n_code_links":1,"syntology":null},{"paper":null,"title":"From Sparse Signal to Smooth Motion: Real-Time Motion Generation with Rolling Prediction Models","date":"2025-01-01","arxiv_id":null,"n_code_links":0,"syntology":null},{"paper":null,"title":"Sentiment trading with large language models","date":"2024-12-26","arxiv_id":"2412.19245","n_code_links":0,"syntology":null},{"paper":null,"title":"RAGONITE: Iterative Retrieval on Induced Databases and Verbalized RDF for Conversational QA over KGs with RAG","date":"2024-12-23","arxiv_id":"2412.17690","n_code_links":0,"syntology":null}],"papers_shown":30,"tasks":[{"task":"/task/language-modelling","name":"Language Modelling","papers":41},{"task":"/task/language-modeling","name":"Language Modeling","papers":28},{"task":"/task/quantization","name":"Quantization","papers":24},{"task":null,"name":"GPU","papers":22},{"task":"/task/large-language-model","name":"Large Language Model","papers":17},{"task":"/task/in-context-learning","name":"In-Context Learning","papers":14},{"task":"/task/question-answering","name":"Question Answering","papers":14},{"task":"/task/text-generation","name":"Text Generation","papers":12},{"task":"/task/model-compression","name":"Model Compression","papers":8},{"task":"/task/retrieval","name":"Retrieval","papers":8},{"task":"/task/fairness","name":"Fairness","papers":7},{"task":"/task/transfer-learning","name":"Transfer Learning","papers":7},{"task":"/task/translation","name":"Translation","papers":7},{"task":"/task/attribute","name":"Attribute","papers":6},{"task":"/task/benchmarking","name":"Benchmarking","papers":6},{"task":"/task/object-detection","name":"Object Detection","papers":6},{"task":"/task/scheduling","name":"Scheduling","papers":6},{"task":"/task/sentence","name":"Sentence","papers":6},{"task":"/task/object-detection-1","name":"object-detection","papers":6},{"task":"/task/computational-efficiency","name":"Computational Efficiency","papers":5}],"tasks_shown":20,"n_tasks":247,"usage_by_year":[{"year":"2022","papers":49},{"year":"2023","papers":104},{"year":"2024","papers":104},{"year":"2025","papers":28}],"row_source":"methods_table","archive":{"source":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","archive_url":"https://paperswithcode.com/method/opt"},"syntology_read_at":"2026-09-24T18:15:14+00:00"}