{"url":"/method/grouped-query-attention","slug":"grouped-query-attention","name":"Grouped-query attention","full_name":"Grouped-query attention","full_name_withheld":false,"description_markdown":"**Grouped-query attention** an interpolation of multi-query and multi-head attention that achieves quality close to multi-head at comparable speed to multi-query attention.","description_state":"present","introduced_year":null,"introduced_by":{"title":null,"paper":null,"first_author":null,"n_authors":0,"url_abs":null,"archive_paper_url":null},"source":{"url":"https://arxiv.org/abs/2305.13245v3","title":"GQA: Training Generalized Multi-Query Transformer Models from Multi-Head Checkpoints","url_on_a_paper_host":true},"code_snippet_url":null,"code_snippet_url_on_a_code_host":false,"categories":[{"area":"General","area_id":"general","collection":"Attention","url":"/methods/category/attention","pwc_aliases":[]}],"n_papers_tagged":19,"archive_num_papers":null,"papers_newest_first":[{"paper":"/paper/hardware-efficient-attention-for-fast","title":"Hardware-Efficient Attention for Fast Decoding","date":"2025-05-27","arxiv_id":"2505.21487","n_code_links":2,"syntology":null},{"paper":null,"title":"Hunyuan-TurboS: Advancing Large Language Models through Mamba-Transformer Synergy and Adaptive Chain-of-Thought","date":"2025-05-21","arxiv_id":"2505.15431","n_code_links":0,"syntology":null},{"paper":"/paper/towards-economical-inference-enabling","title":"Towards Economical Inference: Enabling DeepSeek's Multi-Head Latent Attention in Any Transformer-based LLMs","date":"2025-02-20","arxiv_id":"2502.14837","n_code_links":1,"syntology":{"ran":0,"of":15,"unverified":15,"pointer_only":0}},{"paper":"/paper/fastkv-kv-cache-compression-for-fast-long","title":"FastKV: KV Cache Compression for Fast Long-Context Processing with Token-Selective Propagation","date":"2025-02-03","arxiv_id":"2502.01068","n_code_links":1,"syntology":null},{"paper":null,"title":"Sigma: Differential Rescaling of Query, Key and Value for Efficient Language Models","date":"2025-01-23","arxiv_id":"2501.13629","n_code_links":0,"syntology":null},{"paper":null,"title":"UNComp: Uncertainty-Aware Long-Context Compressor for Efficient Large Language Model Inference","date":"2024-10-04","arxiv_id":"2410.03090","n_code_links":0,"syntology":null},{"paper":null,"title":"Weighted Grouped Query Attention in Transformers","date":"2024-07-15","arxiv_id":"2407.10855","n_code_links":0,"syntology":null},{"paper":"/paper/mlkv-multi-layer-key-value-heads-for-memory","title":"MLKV: Multi-Layer Key-Value Heads for Memory Efficient Transformer Decoding","date":"2024-06-13","arxiv_id":"2406.09297","n_code_links":1,"syntology":null},{"paper":null,"title":"Effectively Compress KV Heads for LLM","date":"2024-06-11","arxiv_id":"2406.07056","n_code_links":0,"syntology":null},{"paper":"/paper/samba-simple-hybrid-state-space-models-for","title":"Samba: Simple Hybrid State Space Models for Efficient Unlimited Context Language Modeling","date":"2024-06-11","arxiv_id":"2406.07522","n_code_links":2,"syntology":{"ran":1,"of":1,"unverified":0,"pointer_only":0}},{"paper":"/paper/reducing-transformer-key-value-cache-size","title":"Reducing Transformer Key-Value Cache Size with Cross-Layer Attention","date":"2024-05-21","arxiv_id":"2405.12981","n_code_links":2,"syntology":null},{"paper":null,"title":"CodeShell Technical Report","date":"2024-03-23","arxiv_id":"2403.15747","n_code_links":0,"syntology":null},{"paper":null,"title":"Vi-Mistral-X: Building a Vietnamese Language Model with Advanced Continual Pre-training","date":"2024-03-20","arxiv_id":"2403.15470","n_code_links":0,"syntology":null},{"paper":"/paper/dynamic-memory-compression-retrofitting-llms","title":"Dynamic Memory Compression: Retrofitting LLMs for Accelerated Inference","date":"2024-03-14","arxiv_id":"2403.09636","n_code_links":1,"syntology":null},{"paper":"/paper/mobilellm-optimizing-sub-billion-parameter","title":"MobileLLM: Optimizing Sub-billion Parameter Language Models for On-Device Use Cases","date":"2024-02-22","arxiv_id":"2402.14905","n_code_links":4,"syntology":{"ran":11,"of":28,"unverified":17,"pointer_only":20}},{"paper":"/paper/mistral-7b","title":"Mistral 7B","date":"2023-10-10","arxiv_id":"2310.06825","n_code_links":6,"syntology":{"ran":9,"of":11,"unverified":2,"pointer_only":1}},{"paper":"/paper/llama-2-open-foundation-and-fine-tuned-chat","title":"Llama 2: Open Foundation and Fine-Tuned Chat Models","date":"2023-07-18","arxiv_id":"2307.09288","n_code_links":19,"syntology":{"ran":31,"of":52,"unverified":21,"pointer_only":16}},{"paper":"/paper/gqa-training-generalized-multi-query","title":"GQA: Training Generalized Multi-Query Transformer Models from Multi-Head Checkpoints","date":"2023-05-22","arxiv_id":"2305.13245","n_code_links":4,"syntology":{"ran":4,"of":5,"unverified":1,"pointer_only":3}},{"paper":"/paper/flashattention-fast-and-memory-efficient","title":"FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness","date":"2022-05-27","arxiv_id":"2205.14135","n_code_links":13,"syntology":{"ran":9,"of":30,"unverified":21,"pointer_only":1}}],"papers_shown":19,"tasks":[{"task":"/task/language-modelling","name":"Language Modelling","papers":6},{"task":"/task/language-modeling","name":"Language Modeling","papers":5},{"task":"/task/large-language-model","name":"Large Language Model","papers":3},{"task":"/task/question-answering","name":"Question Answering","papers":3},{"task":"/task/4k","name":"4k","papers":2},{"task":"/task/arithmetic-reasoning","name":"Arithmetic Reasoning","papers":2},{"task":"/task/chatbot","name":"Chatbot","papers":2},{"task":"/task/code-generation","name":"Code Generation","papers":2},{"task":"/task/decoder","name":"Decoder","papers":2},{"task":null,"name":"GPU","papers":2},{"task":"/task/mamba","name":"Mamba","papers":2},{"task":"/task/math-word-problem-solving","name":"Math Word Problem Solving","papers":2},{"task":"/task/multi-task-language-understanding","name":"Multi-task Language Understanding","papers":2},{"task":"/task/sentence-completion","name":"Sentence Completion","papers":2},{"task":"/task/16k","name":"16k","papers":1},{"task":null,"name":"8k","papers":1},{"task":"/task/common-sense-reasoning","name":"Common Sense Reasoning","papers":1},{"task":"/task/computational-efficiency","name":"Computational Efficiency","papers":1},{"task":"/task/document-classification","name":"Document Classification","papers":1},{"task":"/task/humaneval","name":"HumanEval","papers":1}],"tasks_shown":20,"n_tasks":35,"usage_by_year":[{"year":"2022","papers":1},{"year":"2023","papers":3},{"year":"2024","papers":10},{"year":"2025","papers":5}],"row_source":"embedded","archive":{"source":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","archive_url":"https://paperswithcode.com/method/grouped-query-attention"},"syntology_read_at":"2026-09-24T18:15:14+00:00"}