{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/velora-memory-efficient-training-using-rank-1","title":"VeLoRA: Memory Efficient Training using Rank-1 Sub-Token Projections","arxiv_id":"2405.17991","date":"2024-05-28","proceeding":null,"authors":["Roy Miles","Pradyumna Reddy","Ismail Elezi","Jiankang Deng"],"abstract":"Large language models (LLMs) have recently emerged as powerful tools for tackling many language-processing tasks. Despite their success, training and fine-tuning these models is still far too computationally and memory intensive. In this paper, we identify and characterise the important components needed for effective model convergence using gradient descent. In doing so we find that the intermediate activations used to implement backpropagation can be excessively compressed without incurring any degradation in performance. This result leads us to a cheap and memory-efficient algorithm for both fine-tuning and pre-training LLMs. The proposed algorithm simply divides the tokens up into smaller sub-tokens before projecting them onto a fixed 1-dimensional subspace during the forward pass. These features are then coarsely reconstructed during the backward pass to implement the update rules. We confirm the effectiveness of our algorithm as being complimentary to many state-of-the-art PEFT methods on the VTAB-1k fine-tuning benchmark. Furthermore, we outperform QLoRA for fine-tuning LLaMA and show competitive performance against other memory-efficient pre-training methods on the large-scale C4 dataset.","url_abs":"https://arxiv.org/abs/2405.17991v2","url_pdf":"https://arxiv.org/pdf/2405.17991v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"velora-memory-efficient-training-using-rank-1","repo_url":"https://github.com/roymiles/VeLoRA","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[],"methods":[{"method_slug":"llama","method_name":"LLaMA"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2405.17991","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.17991"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/tloen/alpaca-lora","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/jiaweizzhao/GaLore","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/roymiles/VeLoRA","reach":null}],"summary":{"ran_fixture":1,"ran_draft_wrong":2,"unverified":3},"by_repo_kind":{"official":{"samples":3,"ran":2,"repositories":1},"found_in_text":{"samples":3,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"8ceda444ffb950f1","entry":"functional_velora_linear_fn","repo":"roymiles/velora","repo_kind":"official","path":"velora/velora.py","file_url":"https://github.com/roymiles/velora/blob/HEAD/velora/velora.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8ceda444ffb950f1"}},{"code_sha256_prefix":"a4f5d0414a1cf9c6","entry":"translate_state_dict_key","repo":"tloen/alpaca-lora","repo_kind":"found_in_text","path":"export_state_dict_checkpoint.py","file_url":"https://github.com/tloen/alpaca-lora/blob/HEAD/export_state_dict_checkpoint.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a4f5d0414a1cf9c6"}},{"code_sha256_prefix":"ff49f436863cda50","entry":"velora_linear_fn","repo":"roymiles/velora","repo_kind":"official","path":"velora/velora.py","file_url":"https://github.com/roymiles/velora/blob/HEAD/velora/velora.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ff49f436863cda50"}},{"code_sha256_prefix":"7b5051fa2f69e7b9","entry":"VeLoRA","repo":"roymiles/VeLoRA","repo_kind":"official","path":"velora/velora.py","file_url":"https://github.com/roymiles/VeLoRA/blob/HEAD/velora/velora.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7b5051fa2f69e7b9"}},{"code_sha256_prefix":"a2f49e13f6e13d6b","entry":"permute","repo":"tloen/alpaca-lora","repo_kind":"found_in_text","path":"export_state_dict_checkpoint.py","file_url":"https://github.com/tloen/alpaca-lora/blob/HEAD/export_state_dict_checkpoint.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a2f49e13f6e13d6b"}},{"code_sha256_prefix":"6e275f232bbfb9b6","entry":"unpermute","repo":"tloen/alpaca-lora","repo_kind":"found_in_text","path":"export_state_dict_checkpoint.py","file_url":"https://github.com/tloen/alpaca-lora/blob/HEAD/export_state_dict_checkpoint.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6e275f232bbfb9b6"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}