{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/flash-normalization-fast-rmsnorm-for-llms","title":"Flash normalization: fast RMSNorm for LLMs","arxiv_id":"2407.09577","date":"2024-07-12","proceeding":null,"authors":["Nils Graef","Matthew Clapp","Andrew Wasielewski"],"abstract":"RMSNorm is used by many LLMs such as Llama, Mistral, and OpenELM. This paper details FlashNorm, which is an exact but faster implementation of RMSNorm followed by linear layers. See https://huggingface.co/open-machine/FlashNorm for code and more transformer tricks.","url_abs":"https://arxiv.org/abs/2407.09577v1","url_pdf":"https://arxiv.org/pdf/2407.09577v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"flash-normalization-fast-rmsnorm-for-llms","repo_url":"https://github.com/openmachine-ai/transformer-tricks","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[],"methods":[{"method_slug":"rmsnorm","method_name":"RMSNorm"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2407.09577","mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}