{"url":"/method/lion","slug":"lion","name":"Lion","full_name":"Evolved Sign Momentum","full_name_withheld":false,"description_markdown":"The Lion optimizer is discovered by symbolic program search. It is more memory-efficient than most adaptive optimizers as it only needs to momentum. The update of Lion is produced by the sign function.","description_state":"present","introduced_year":null,"introduced_by":{"title":null,"paper":null,"first_author":null,"n_authors":0,"url_abs":null,"archive_paper_url":null},"source":{"url":null,"title":null,"url_on_a_paper_host":false},"code_snippet_url":"https://github.com/google/automl/blob/ecbc7a3b5ab1944d5f39a80a21fd07b7606583f0/lion/lion_pytorch.py","code_snippet_url_on_a_code_host":true,"categories":[{"area":"General","area_id":"general","collection":"Optimization","url":"/methods/category/optimization","pwc_aliases":[]}],"n_papers_tagged":32,"archive_num_papers":32,"papers_newest_first":[{"paper":null,"title":"Towards Efficient and Effective Alignment of Large Language Models","date":"2025-06-11","arxiv_id":"2506.09329","n_code_links":0,"syntology":null},{"paper":null,"title":"Lions and Muons: Optimization via Stochastic Frank-Wolfe","date":"2025-06-04","arxiv_id":"2506.04192","n_code_links":0,"syntology":null},{"paper":"/paper/linear-attention-for-efficient-bidirectional","title":"Linear Attention for Efficient Bidirectional Sequence Modeling","date":"2025-02-22","arxiv_id":"2502.16249","n_code_links":1,"syntology":{"ran":14,"of":14,"unverified":0,"pointer_only":14}},{"paper":null,"title":"AdaGC: Improving Training Stability for Large Language Model Pretraining","date":"2025-02-16","arxiv_id":"2502.11034","n_code_links":0,"syntology":null},{"paper":"/paper/connections-between-schedule-free-optimizers","title":"Connections between Schedule-Free Optimizers, AdEMAMix, and Accelerated SGD Variants","date":"2025-02-04","arxiv_id":"2502.02431","n_code_links":1,"syntology":{"ran":1,"of":1,"unverified":0,"pointer_only":0}},{"paper":null,"title":"How Memory in Optimization Algorithms Implicitly Modifies the Loss","date":"2025-02-04","arxiv_id":"2502.02132","n_code_links":0,"syntology":null},{"paper":"/paper/grams-gradient-descent-with-adaptive-momentum","title":"Grams: Gradient Descent with Adaptive Momentum Scaling","date":"2024-12-22","arxiv_id":"2412.17107","n_code_links":1,"syntology":{"ran":1,"of":1,"unverified":0,"pointer_only":0}},{"paper":null,"title":"Lion Cub: Minimizing Communication Overhead in Distributed Lion","date":"2024-11-25","arxiv_id":"2411.16462","n_code_links":0,"syntology":null},{"paper":"/paper/mars-unleashing-the-power-of-variance","title":"MARS: Unleashing the Power of Variance Reduction for Training Large Models","date":"2024-11-15","arxiv_id":"2411.10438","n_code_links":2,"syntology":{"ran":3,"of":5,"unverified":2,"pointer_only":1}},{"paper":null,"title":"Convergence Rate Analysis of LION","date":"2024-11-12","arxiv_id":"2411.07724","n_code_links":0,"syntology":null},{"paper":null,"title":"TalkMosaic: Interactive PhotoMosaic with Multi-modal LLM Q&A Interactions","date":"2024-09-20","arxiv_id":"2409.13941","n_code_links":0,"syntology":null},{"paper":null,"title":"Learning Spatially-Aware Language and Audio Embeddings","date":"2024-09-17","arxiv_id":"2409.11369","n_code_links":0,"syntology":null},{"paper":"/paper/lion-linear-group-rnn-for-3d-object-detection","title":"LION: Linear Group RNN for 3D Object Detection in Point Clouds","date":"2024-07-25","arxiv_id":"2407.18232","n_code_links":1,"syntology":{"ran":0,"of":7,"unverified":7,"pointer_only":0}},{"paper":null,"title":"Deconstructing What Makes a Good Optimizer for Language Models","date":"2024-07-10","arxiv_id":"2407.07972","n_code_links":0,"syntology":null},{"paper":null,"title":"Dynamic Multi-Objective Lion Swarm Optimization with Multi-strategy Fusion: An application in 6R robot trajectory planning","date":"2024-05-31","arxiv_id":"2406.00114","n_code_links":0,"syntology":null},{"paper":null,"title":"Surge Phenomenon in Optimal Learning Rate and Batch Size Scaling","date":"2024-05-23","arxiv_id":"2405.14578","n_code_links":0,"syntology":null},{"paper":"/paper/audio-visual-speech-recognition-based-on","title":"Audio-Visual Speech Recognition based on Regulated Transformer and Spatio-Temporal Fusion Strategy for Driver Assistive Systems","date":"2024-05-09","arxiv_id":null,"n_code_links":1,"syntology":null},{"paper":null,"title":"Communication Efficient Distributed Training with Distributed Lion","date":"2024-03-30","arxiv_id":"2404.00438","n_code_links":0,"syntology":null},{"paper":"/paper/fedlion-faster-adaptive-federated","title":"FedLion: Faster Adaptive Federated Optimization with Fewer Communication","date":"2024-02-15","arxiv_id":"2402.09941","n_code_links":1,"syntology":{"ran":1,"of":1,"unverified":0,"pointer_only":1}},{"paper":null,"title":"MADA: Meta-Adaptive Optimizers through hyper-gradient Descent","date":"2024-01-17","arxiv_id":"2401.08893","n_code_links":0,"syntology":null},{"paper":"/paper/collie-collaborative-training-of-large","title":"CoLLiE: Collaborative Training of Large Language Models in an Efficient Way","date":"2023-12-01","arxiv_id":"2312.00407","n_code_links":1,"syntology":{"ran":4,"of":7,"unverified":3,"pointer_only":0}},{"paper":null,"title":"Rethinking SIGN Training: Provable Nonconvex Acceleration without First- and Second-Order Gradient Lipschitz","date":"2023-10-23","arxiv_id":"2310.14616","n_code_links":0,"syntology":null},{"paper":null,"title":"QFT: Quantized Full-parameter Tuning of LLMs with Affordable Resources","date":"2023-10-11","arxiv_id":"2310.07147","n_code_links":0,"syntology":null},{"paper":null,"title":"Lion Secretly Solves Constrained Optimization: As Lyapunov Predicts","date":"2023-10-09","arxiv_id":"2310.05898","n_code_links":0,"syntology":null},{"paper":"/paper/handwritten-stenography-recognition-and-the","title":"Handwritten Stenography Recognition and the LION Dataset","date":"2023-08-15","arxiv_id":"2308.07799","n_code_links":1,"syntology":null},{"paper":null,"title":"Stochastic Subgradient Methods with Guaranteed Global Stability in Nonsmooth Nonconvex Optimization","date":"2023-07-19","arxiv_id":"2307.10053","n_code_links":0,"syntology":null},{"paper":null,"title":"Automatic Trade-off Adaptation in Offline RL","date":"2023-06-16","arxiv_id":"2306.09744","n_code_links":0,"syntology":null},{"paper":"/paper/rotational-optimizers-simple-robust-dnn","title":"Rotational Equilibrium: How Weight Decay Balances Learning Across Neural Networks","date":"2023-05-26","arxiv_id":"2305.17212","n_code_links":2,"syntology":{"ran":0,"of":6,"unverified":6,"pointer_only":0}},{"paper":null,"title":"LION: Implicit Vision Prompt Tuning","date":"2023-03-17","arxiv_id":"2303.09992","n_code_links":0,"syntology":null},{"paper":null,"title":"Evolving Deep Neural Network by Customized Moth Flame Optimization Algorithm for Underwater Targets Recognition","date":"2023-02-16","arxiv_id":"2303.00922","n_code_links":0,"syntology":null}],"papers_shown":30,"tasks":[{"task":"/task/language-modeling","name":"Language Modeling","papers":4},{"task":"/task/language-modelling","name":"Language Modelling","papers":4},{"task":"/task/quantization","name":"Quantization","papers":3},{"task":"/task/contrastive-learning","name":"Contrastive Learning","papers":2},{"task":null,"name":"GPU","papers":2},{"task":null,"name":"Position","papers":2},{"task":"/task/stochastic-optimization","name":"Stochastic Optimization","papers":2},{"task":"/task/parameter-efficient-fine-tuning","name":"parameter-efficient fine-tuning","papers":2},{"task":"/task/3d-object-detection","name":"3D Object Detection","papers":1},{"task":"/task/attribute","name":"Attribute","papers":1},{"task":"/task/audio-visual-speech-recognition","name":"Audio-Visual Speech Recognition","papers":1},{"task":"/task/diversity","name":"Diversity","papers":1},{"task":"/task/federated-learning","name":"Federated Learning","papers":1},{"task":"/task/handwritten-text-recognition","name":"Handwritten Text Recognition","papers":1},{"task":"/task/image-classification","name":"Image Classification","papers":1},{"task":"/task/l2-regularization","name":"L2 Regularization","papers":1},{"task":"/task/lambada","name":"LAMBADA","papers":1},{"task":"/task/large-language-model","name":"Large Language Model","papers":1},{"task":"/task/lip-reading","name":"Lip Reading","papers":1},{"task":"/task/lipreading","name":"Lipreading","papers":1}],"tasks_shown":20,"n_tasks":38,"usage_by_year":[{"year":"2023","papers":12},{"year":"2024","papers":14},{"year":"2025","papers":6}],"row_source":"methods_table","archive":{"source":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","archive_url":"https://paperswithcode.com/method/lion"},"syntology_read_at":"2026-09-24T18:15:14+00:00"}