{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/loretta-low-rank-economic-tensor-train","title":"LoRETTA: Low-Rank Economic Tensor-Train Adaptation for Ultra-Low-Parameter Fine-Tuning of Large Language Models","arxiv_id":"2402.11417","date":"2024-02-18","proceeding":null,"authors":["Yifan Yang","Jiajun Zhou","Ngai Wong","Zheng Zhang"],"abstract":"Various parameter-efficient fine-tuning (PEFT) techniques have been proposed to enable computationally efficient fine-tuning while maintaining model performance. However, existing PEFT methods are still limited by the growing number of trainable parameters with the rapid deployment of Large Language Models (LLMs). To address this challenge, we present LoRETTA, an ultra-parameter-efficient framework that significantly reduces trainable parameters through tensor-train decomposition. Specifically, we propose two methods, named {LoRETTA}$_{adp}$ and {LoRETTA}$_{rep}$. The former employs tensorized adapters, offering a high-performance yet lightweight approach for the fine-tuning of LLMs. The latter emphasizes fine-tuning via weight parameterization with a set of small tensor factors. LoRETTA achieves comparable or better performance than most widely used PEFT methods with up to $100\\times$ fewer parameters on the LLaMA-2-7B models. Furthermore, empirical results demonstrate that the proposed method effectively improves training efficiency, enjoys better multi-task learning performance, and enhances the anti-overfitting capability. Plug-and-play codes built upon the Huggingface framework and PEFT library will be released.","url_abs":"https://arxiv.org/abs/2402.11417v1","url_pdf":"https://arxiv.org/pdf/2402.11417v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"loretta-low-rank-economic-tensor-train","repo_url":"https://github.com/yifanycc/loretta","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"GPL-3.0"}}],"tasks":[{"task_slug":"multi-task-learning","task_name":"Multi-Task Learning"},{"task_slug":"parameter-efficient-fine-tuning","task_name":"parameter-efficient fine-tuning"}],"methods":[{"method_slug":null,"method_name":"Library"},{"method_slug":"set","method_name":"SET"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2402.11417","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.11417"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yifanycc/loretta","reach":{"status":"ok","spdx":"GPL-3.0"}}],"summary":{"ran":2,"ran_draft_wrong":2,"unverified":4},"by_repo_kind":{"official":{"samples":8,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":8,"samples":[{"code_sha256_prefix":"665af05400a43244","entry":"build_message","repo":"yifanycc/loretta","repo_kind":"official","path":"large_models/message.py","file_url":"https://github.com/yifanycc/loretta/blob/HEAD/large_models/message.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"665af05400a43244"}},{"code_sha256_prefix":"b5adfd6ac99a1b18","entry":"find_module","repo":"yifanycc/loretta","repo_kind":"official","path":"large_models/lora.py","file_url":"https://github.com/yifanycc/loretta/blob/HEAD/large_models/lora.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"b5adfd6ac99a1b18"}},{"code_sha256_prefix":"918fc46b5088d174","entry":"get_parameter_number","repo":"yifanycc/loretta","repo_kind":"official","path":"bert_model/run_glue_v5.py","file_url":"https://github.com/yifanycc/loretta/blob/HEAD/bert_model/run_glue_v5.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"918fc46b5088d174"}},{"code_sha256_prefix":"e7e75981cb464788","entry":"normalize_answer","repo":"yifanycc/loretta","repo_kind":"official","path":"bert_model/metrics.py","file_url":"https://github.com/yifanycc/loretta/blob/HEAD/bert_model/metrics.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"code_sha256_prefix":"bdb460cb247501dc","entry":"attn_forward_hook","repo":"yifanycc/loretta","repo_kind":"official","path":"large_models/prefix.py","file_url":"https://github.com/yifanycc/loretta/blob/HEAD/large_models/prefix.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"bdb460cb247501dc"}},{"code_sha256_prefix":"84c658f2c9252878","entry":"calculate_metric","repo":"yifanycc/loretta","repo_kind":"official","path":"bert_model/metrics.py","file_url":"https://github.com/yifanycc/loretta/blob/HEAD/bert_model/metrics.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"84c658f2c9252878"}},{"code_sha256_prefix":"b989d4bce26f77ce","entry":"f1","repo":"yifanycc/loretta","repo_kind":"official","path":"bert_model/metrics.py","file_url":"https://github.com/yifanycc/loretta/blob/HEAD/bert_model/metrics.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"b989d4bce26f77ce"}},{"code_sha256_prefix":"84920a8095dca4fa","entry":"prepare_inputs_for_generation","repo":"yifanycc/loretta","repo_kind":"official","path":"large_models/prefix.py","file_url":"https://github.com/yifanycc/loretta/blob/HEAD/large_models/prefix.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"84920a8095dca4fa"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}