{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2510-09160","title":"Efficient Resource-Constrained Training of Transformers via Subspace Optimization","arxiv_id":"2510.09160","date":"2025-10-10","proceeding":"ICLR","authors":["Le-Trung Nguyen","Enzo Tartaglione","Van-Tam Nguyen"],"abstract":"As AI increasingly shapes daily life, energy consumption and data privacy have become pressing concerns. On-device learning trains models directly on edge devices, cutting energy consumption and safeguarding data privacy. However, the expanding scale of modern neural networks creates a major obstacle for on-device training. Although prior work has concentrated on compact convolutional architectures, we instead apply subspace-based training to transformer models. Motivated by the idea that a model's essential information lies in a fixed subspace, we introduce Weight-Activation Subspace Iteration (WASI), a method that mitigates the memory bottleneck of backpropagation and boosts inference efficiency in transformer models by restricting training to this subspace. Our results demonstrate that WASI maintains accuracy comparable to vanilla training while reducing memory usage by up to $62\\times$ and computational cost (FLOPs) by up to $2\\times$. On a Raspberry Pi 5, WASI achieves roughly $1.4\\times$ faster training and inference than vanilla training. The code is available at https://github.com/Le-TrungNguyen/ICLR2026-WASI.git.","url_abs":"https://arxiv.org/abs/2510.09160","url_pdf":"https://arxiv.org/pdf/2510.09160","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2510.09160","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2510.09160"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/Le-TrungNguyen/ICLR2026-WASI","reach":null}],"summary":{"unverified":5},"by_repo_kind":{"found_in_text":{"samples":5,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":5,"samples":[{"code_sha256_prefix":"85673bd2f016892f","entry":"Linear_WASI_op","repo":"Le-TrungNguyen/ICLR2026-WASI","repo_kind":"found_in_text","path":"main/custom_op/linear/linear_WASI.py","file_url":"https://github.com/Le-TrungNguyen/ICLR2026-WASI/blob/HEAD/main/custom_op/linear/linear_WASI.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"85673bd2f016892f"}},{"code_sha256_prefix":"242e22557809363f","entry":"_get_wasi_streams","repo":"Le-TrungNguyen/ICLR2026-WASI","repo_kind":"found_in_text","path":"main/custom_op/linear/linear_WASI.py","file_url":"https://github.com/Le-TrungNguyen/ICLR2026-WASI/blob/HEAD/main/custom_op/linear/linear_WASI.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"242e22557809363f"}},{"code_sha256_prefix":"b6f2df73350c1d74","entry":"_low_rank_linear","repo":"Le-TrungNguyen/ICLR2026-WASI","repo_kind":"found_in_text","path":"main/custom_op/linear/linear_WASI.py","file_url":"https://github.com/Le-TrungNguyen/ICLR2026-WASI/blob/HEAD/main/custom_op/linear/linear_WASI.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b6f2df73350c1d74"}},{"code_sha256_prefix":"21fde2690444842c","entry":"_wasi_grad_weight_3d","repo":"Le-TrungNguyen/ICLR2026-WASI","repo_kind":"found_in_text","path":"main/custom_op/linear/linear_WASI.py","file_url":"https://github.com/Le-TrungNguyen/ICLR2026-WASI/blob/HEAD/main/custom_op/linear/linear_WASI.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"21fde2690444842c"}},{"code_sha256_prefix":"fc79ed64e6c1d90e","entry":"_wasi_grad_weight_4d","repo":"Le-TrungNguyen/ICLR2026-WASI","repo_kind":"found_in_text","path":"main/custom_op/linear/linear_WASI.py","file_url":"https://github.com/Le-TrungNguyen/ICLR2026-WASI/blob/HEAD/main/custom_op/linear/linear_WASI.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fc79ed64e6c1d90e"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.LG","source":"arxiv_api"},"syntology_extracted_results":null}