{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/investigating-low-rank-training-in","title":"Investigating Low-Rank Training in Transformer Language Models: Efficiency and Scaling Analysis","arxiv_id":"2407.09835","date":"2024-07-13","proceeding":null,"authors":["Xiuying Wei","Skander Moalla","Razvan Pascanu","Caglar Gulcehre"],"abstract":"State-of-the-art LLMs often rely on scale with high computational costs, which has sparked a research agenda to reduce parameter counts and costs without significantly impacting performance. Our study focuses on Transformer-based LLMs, specifically applying low-rank parametrization to the computationally intensive feedforward networks (FFNs), which are less studied than attention blocks. In contrast to previous works, (i) we explore low-rank parametrization at scale, up to 1.3B parameters; (ii) within Transformer language models rather than convolutional architectures; and (iii) starting from training from scratch. Experiments on the large RefinedWeb dataset show that low-rank parametrization is both efficient (e.g., 2.6$\\times$ FFN speed-up with 32\\% parameters) and effective during training. Interestingly, these structured FFNs exhibit steeper scaling curves than the original models. Motivated by this finding, we develop the wide and structured networks surpassing the current medium-sized and large-sized Transformer in perplexity and throughput performance. Our code is available at https://github.com/CLAIRE-Labo/StructuredFFN/tree/main.","url_abs":"https://arxiv.org/abs/2407.09835v2","url_pdf":"https://arxiv.org/pdf/2407.09835v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"investigating-low-rank-training-in","repo_url":"https://github.com/claire-labo/structuredffn","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2407.09835","atlas_url":"https://app.syntology.ai/?focus=2407.09835","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.09835"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/CLAIRE-Labo/StructuredFFN","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/claire-labo/structuredffn","reach":{"status":"ok"}}],"summary":{"ran":3,"unverified":3},"by_repo_kind":{"official":{"samples":6,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":6,"samples":[{"code_sha256_prefix":"1da3d260d5f0d416","entry":"benchmark_train","repo":"CLAIRE-Labo/StructuredFFN","repo_kind":"official","path":"src/benchmark_eff/benchmark_model_train.py","file_url":"https://github.com/CLAIRE-Labo/StructuredFFN/blob/HEAD/src/benchmark_eff/benchmark_model_train.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1da3d260d5f0d416"}},{"code_sha256_prefix":"6dc0ac45d5bcd810","entry":"find_max_batch_size","repo":"CLAIRE-Labo/StructuredFFN","repo_kind":"official","path":"src/benchmark_eff/cac_batch.py","file_url":"https://github.com/CLAIRE-Labo/StructuredFFN/blob/HEAD/src/benchmark_eff/cac_batch.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6dc0ac45d5bcd810"}},{"code_sha256_prefix":"55c08b060f32da53","entry":"func","repo":"CLAIRE-Labo/StructuredFFN","repo_kind":"official","path":"src/benchmark_eff/cac_batch.py","file_url":"https://github.com/CLAIRE-Labo/StructuredFFN/blob/HEAD/src/benchmark_eff/cac_batch.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"55c08b060f32da53"}},{"code_sha256_prefix":"948efcbcaf169671","entry":"benchmark_infer","repo":"CLAIRE-Labo/StructuredFFN","repo_kind":"official","path":"src/benchmark_eff/benchmark_model_infer.py","file_url":"https://github.com/CLAIRE-Labo/StructuredFFN/blob/HEAD/src/benchmark_eff/benchmark_model_infer.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"948efcbcaf169671"}},{"code_sha256_prefix":"4babe6e67c1b55ad","entry":"group_context","repo":"CLAIRE-Labo/StructuredFFN","repo_kind":"official","path":"src/utils/refinedweb_llama.py","file_url":"https://github.com/CLAIRE-Labo/StructuredFFN/blob/HEAD/src/utils/refinedweb_llama.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4babe6e67c1b55ad"}},{"code_sha256_prefix":"6c25271fe6f1f40f","entry":"tokenize","repo":"CLAIRE-Labo/StructuredFFN","repo_kind":"official","path":"src/utils/refinedweb_llama.py","file_url":"https://github.com/CLAIRE-Labo/StructuredFFN/blob/HEAD/src/utils/refinedweb_llama.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6c25271fe6f1f40f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}