{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/griffin-mixing-gated-linear-recurrences-with","title":"Griffin: Mixing Gated Linear Recurrences with Local Attention for Efficient Language Models","arxiv_id":"2402.19427","date":"2024-02-29","proceeding":null,"authors":["Soham De","Samuel L. Smith","Anushan Fernando","Aleksandar Botev","George Cristian-Muraru","Albert Gu","Ruba Haroun","Leonard Berrada","Yutian Chen","Srivatsan Srinivasan","Guillaume Desjardins","Arnaud Doucet","David Budden","Yee Whye Teh","Razvan Pascanu","Nando de Freitas","Caglar Gulcehre"],"abstract":"Recurrent neural networks (RNNs) have fast inference and scale efficiently on long sequences, but they are difficult to train and hard to scale. We propose Hawk, an RNN with gated linear recurrences, and Griffin, a hybrid model that mixes gated linear recurrences with local attention. Hawk exceeds the reported performance of Mamba on downstream tasks, while Griffin matches the performance of Llama-2 despite being trained on over 6 times fewer tokens. We also show that Griffin can extrapolate on sequences significantly longer than those seen during training. Our models match the hardware efficiency of Transformers during training, and during inference they have lower latency and significantly higher throughput. We scale Griffin up to 14B parameters, and explain how to shard our models for efficient distributed training.","url_abs":"https://arxiv.org/abs/2402.19427v1","url_pdf":"https://arxiv.org/pdf/2402.19427v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"griffin-mixing-gated-linear-recurrences-with","repo_url":"https://github.com/google-deepmind/recurrentgemma","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"griffin-mixing-gated-linear-recurrences-with","repo_url":"https://github.com/google-gemini/gemma-cookbook","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"griffin-mixing-gated-linear-recurrences-with","repo_url":"https://github.com/knotgrass/Griffin","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"griffin-mixing-gated-linear-recurrences-with","repo_url":"https://github.com/proger/hippogriff","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"mamba","task_name":"Mamba"}],"methods":[{"method_slug":"glu","method_name":"Gated Linear Unit"},{"method_slug":"global-local-attention","method_name":"Global-Local Attention"},{"method_slug":"multi-query-attention","method_name":"Multi-Query Attention"},{"method_slug":"rmsnorm","method_name":"RMSNorm"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2402.19427","atlas_url":"https://app.syntology.ai/?focus=2402.19427","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.19427"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/google-deepmind/recurrentgemma","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/knotgrass/Griffin","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/google-gemini/gemma-cookbook","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/proger/hippogriff","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":10},"by_repo_kind":{"listed":{"samples":10,"ran":10,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"28867e57625ea792","entry":"export_for_matformer_lab","repo":"google-gemini/gemma-cookbook","repo_kind":"listed","path":"Guides/Gemma-3n-Submodels/custom_slicing_configs.py","file_url":"https://github.com/google-gemini/gemma-cookbook/blob/HEAD/Guides/Gemma-3n-Submodels/custom_slicing_configs.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"28867e57625ea792"}},{"code_sha256_prefix":"a44998324fc779df","entry":"extract_substring","repo":"google-gemini/gemma-cookbook","repo_kind":"listed","path":"Demos/spoken-language-tasks/k-mail-replier/k_mail_replier/models/gemma.py","file_url":"https://github.com/google-gemini/gemma-cookbook/blob/HEAD/Demos/spoken-language-tasks/k-mail-replier/k_mail_replier/models/gemma.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a44998324fc779df"}},{"code_sha256_prefix":"ee6bd7400c685050","entry":"get_config_for_deployment","repo":"google-gemini/gemma-cookbook","repo_kind":"listed","path":"Guides/Gemma-3n-Submodels/custom_slicing_configs.py","file_url":"https://github.com/google-gemini/gemma-cookbook/blob/HEAD/Guides/Gemma-3n-Submodels/custom_slicing_configs.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ee6bd7400c685050"}},{"code_sha256_prefix":"70b3d78ffd9316ee","entry":"list_checkpoints","repo":"proger/hippogriff","repo_kind":"listed","path":"train_init.py","file_url":"https://github.com/proger/hippogriff/blob/HEAD/train_init.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"70b3d78ffd9316ee"}},{"code_sha256_prefix":"1d8a8cadca2d8282","entry":"load_checkpoint","repo":"proger/hippogriff","repo_kind":"listed","path":"train_init.py","file_url":"https://github.com/proger/hippogriff/blob/HEAD/train_init.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1d8a8cadca2d8282"}},{"code_sha256_prefix":"903df3e710bd40d2","entry":"print_weights","repo":"proger/hippogriff","repo_kind":"listed","path":"train_diagnostics.py","file_url":"https://github.com/proger/hippogriff/blob/HEAD/train_diagnostics.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"903df3e710bd40d2"}},{"code_sha256_prefix":"1a3e30d8ca827338","entry":"summarize_gradients","repo":"proger/hippogriff","repo_kind":"listed","path":"train_diagnostics.py","file_url":"https://github.com/proger/hippogriff/blob/HEAD/train_diagnostics.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1a3e30d8ca827338"}},{"code_sha256_prefix":"3366f12b87851a4f","entry":"trim_response","repo":"google-gemini/gemma-cookbook","repo_kind":"listed","path":"Demos/business-email-assistant/email-processing-webapp/models/gemma.py","file_url":"https://github.com/google-gemini/gemma-cookbook/blob/HEAD/Demos/business-email-assistant/email-processing-webapp/models/gemma.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3366f12b87851a4f"}},{"code_sha256_prefix":"f820e6681777dded","entry":"trim_response","repo":"google-gemini/gemma-cookbook","repo_kind":"listed","path":"Demos/personal-code-assistant/gemma-web-service/gemma_service/gemma_model.py","file_url":"https://github.com/google-gemini/gemma-cookbook/blob/HEAD/Demos/personal-code-assistant/gemma-web-service/gemma_service/gemma_model.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f820e6681777dded"}},{"code_sha256_prefix":"37899b9ccd873f9a","entry":"validate_config","repo":"google-gemini/gemma-cookbook","repo_kind":"listed","path":"Guides/Gemma-3n-Submodels/custom_slicing_configs.py","file_url":"https://github.com/google-gemini/gemma-cookbook/blob/HEAD/Guides/Gemma-3n-Submodels/custom_slicing_configs.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"37899b9ccd873f9a"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}