{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/compute-better-spent-replacing-dense-layers","title":"Compute Better Spent: Replacing Dense Layers with Structured Matrices","arxiv_id":"2406.06248","date":"2024-06-10","proceeding":null,"authors":["Shikai Qiu","Andres Potapczynski","Marc Finzi","Micah Goldblum","Andrew Gordon Wilson"],"abstract":"Dense linear layers are the dominant computational bottleneck in foundation models. Identifying more efficient alternatives to dense matrices has enormous potential for building more compute-efficient models, as exemplified by the success of convolutional networks in the image domain. In this work, we systematically explore structured matrices as replacements for dense matrices. We show that different structures often require drastically different initialization scales and learning rates, which are crucial to performance, especially as models scale. Using insights from the Maximal Update Parameterization, we determine the optimal scaling for initialization and learning rates of these unconventional layers. Finally, we measure the scaling laws of different structures to compare how quickly their performance improves with compute. We propose a novel matrix family containing Monarch matrices, the Block Tensor-Train (BTT), which we show performs better than dense matrices for the same compute on multiple tasks. On CIFAR-10/100 with augmentation, BTT achieves exponentially lower training loss than dense when training MLPs and ViTs. BTT matches dense ViT-S/32 performance on ImageNet-1k with 3.8 times less compute and is more efficient than dense for training small GPT-2 language models.","url_abs":"https://arxiv.org/abs/2406.06248v1","url_pdf":"https://arxiv.org/pdf/2406.06248v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"compute-better-spent-replacing-dense-layers","repo_url":"https://github.com/shikaiqiu/compute-better-spent","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"cosine-annealing","method_name":"Cosine Annealing"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"discriminative-fine-tuning","method_name":"Discriminative Fine-Tuning"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt-2","method_name":"GPT-2"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-cosine-annealing","method_name":"Linear Warmup With Cosine Annealing"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2406.06248","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.06248"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/shikaiqiu/compute-better-spent","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":4,"unverified":1},"by_repo_kind":{"official":{"samples":5,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"402d7dec03e070bb","entry":"construct_configs","repo":"shikaiqiu/compute-better-spent","repo_kind":"official","path":"model/gpt_fns.py","file_url":"https://github.com/shikaiqiu/compute-better-spent/blob/HEAD/model/gpt_fns.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"402d7dec03e070bb"}},{"code_sha256_prefix":"4e894cdc17d93f2c","entry":"eval_model","repo":"shikaiqiu/compute-better-spent","repo_kind":"official","path":"model/fns.py","file_url":"https://github.com/shikaiqiu/compute-better-spent/blob/HEAD/model/fns.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4e894cdc17d93f2c"}},{"code_sha256_prefix":"b8b4a595e35ec125","entry":"get_epoch","repo":"shikaiqiu/compute-better-spent","repo_kind":"official","path":"model/gpt_fns.py","file_url":"https://github.com/shikaiqiu/compute-better-spent/blob/HEAD/model/gpt_fns.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b8b4a595e35ec125"}},{"code_sha256_prefix":"c5e8ad6242953e5e","entry":"train_epoch","repo":"shikaiqiu/compute-better-spent","repo_kind":"official","path":"model/fns.py","file_url":"https://github.com/shikaiqiu/compute-better-spent/blob/HEAD/model/fns.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c5e8ad6242953e5e"}},{"code_sha256_prefix":"9af0eec9eafe9422","entry":"init_dist_process","repo":"shikaiqiu/compute-better-spent","repo_kind":"official","path":"model/gpt_fns.py","file_url":"https://github.com/shikaiqiu/compute-better-spent/blob/HEAD/model/gpt_fns.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9af0eec9eafe9422"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}