{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/adazeta-adaptive-zeroth-order-tensor-train","title":"AdaZeta: Adaptive Zeroth-Order Tensor-Train Adaption for Memory-Efficient Large Language Models Fine-Tuning","arxiv_id":"2406.18060","date":"2024-06-26","proceeding":null,"authors":["Yifan Yang","Kai Zhen","Ershad Banijamal","Athanasios Mouchtaris","Zheng Zhang"],"abstract":"Fine-tuning large language models (LLMs) has achieved remarkable performance across various natural language processing tasks, yet it demands more and more memory as model sizes keep growing. To address this issue, the recently proposed Memory-efficient Zeroth-order (MeZO) methods attempt to fine-tune LLMs using only forward passes, thereby avoiding the need for a backpropagation graph. However, significant performance drops and a high risk of divergence have limited their widespread adoption. In this paper, we propose the Adaptive Zeroth-order Tensor-Train Adaption (AdaZeta) framework, specifically designed to improve the performance and convergence of the ZO methods. To enhance dimension-dependent ZO estimation accuracy, we introduce a fast-forward, low-parameter tensorized adapter. To tackle the frequently observed divergence issue in large-scale ZO fine-tuning tasks, we propose an adaptive query number schedule that guarantees convergence. Detailed theoretical analysis and extensive experimental results on Roberta-Large and Llama-2-7B models substantiate the efficacy of our AdaZeta framework in terms of accuracy, memory efficiency, and convergence speed.","url_abs":"https://arxiv.org/abs/2406.18060v3","url_pdf":"https://arxiv.org/pdf/2406.18060v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"adazeta-adaptive-zeroth-order-tensor-train","repo_url":"https://github.com/yifanycc/adazeta","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2406.18060","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.18060"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yifanycc/adazeta","reach":null}],"summary":{"ran":5,"unverified":4},"by_repo_kind":{"official":{"samples":9,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":9,"samples":[{"code_sha256_prefix":"3bdbc3383c2ef5c7","entry":"AdapterLayer","repo":"yifanycc/adazeta","repo_kind":"official","path":"large_models/loretta/tuners/loretta_adp.py","file_url":"https://github.com/yifanycc/adazeta/blob/HEAD/large_models/loretta/tuners/loretta_adp.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"3bdbc3383c2ef5c7"}},{"code_sha256_prefix":"201eba09eae2d29d","entry":"TT_forward_quant","repo":"yifanycc/adazeta","repo_kind":"official","path":"large_models/loretta/tuners/loretta_adp.py","file_url":"https://github.com/yifanycc/adazeta/blob/HEAD/large_models/loretta/tuners/loretta_adp.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"201eba09eae2d29d"}},{"code_sha256_prefix":"1cc70ec1831f942f","entry":"TensorTrain","repo":"yifanycc/adazeta","repo_kind":"official","path":"large_models/loretta/tuners/loretta_adp.py","file_url":"https://github.com/yifanycc/adazeta/blob/HEAD/large_models/loretta/tuners/loretta_adp.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"1cc70ec1831f942f"}},{"code_sha256_prefix":"92ad4c13176c2f33","entry":"TensorizedLinear_module","repo":"yifanycc/adazeta","repo_kind":"official","path":"large_models/loretta/tuners/loretta_adp.py","file_url":"https://github.com/yifanycc/adazeta/blob/HEAD/large_models/loretta/tuners/loretta_adp.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"92ad4c13176c2f33"}},{"code_sha256_prefix":"e52c09d7918640f3","entry":"quantize","repo":"yifanycc/adazeta","repo_kind":"official","path":"large_models/loretta/tuners/loretta_adp.py","file_url":"https://github.com/yifanycc/adazeta/blob/HEAD/large_models/loretta/tuners/loretta_adp.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"e52c09d7918640f3"}},{"code_sha256_prefix":"3c197a7557ec2f72","entry":"Linear","repo":"yifanycc/adazeta","repo_kind":"official","path":"large_models/loretta/tuners/loretta_adp.py","file_url":"https://github.com/yifanycc/adazeta/blob/HEAD/large_models/loretta/tuners/loretta_adp.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"3c197a7557ec2f72"}},{"code_sha256_prefix":"99f1245ca18ffc06","entry":"LorettaAdpModel","repo":"yifanycc/adazeta","repo_kind":"official","path":"large_models/loretta/tuners/loretta_adp.py","file_url":"https://github.com/yifanycc/adazeta/blob/HEAD/large_models/loretta/tuners/loretta_adp.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"99f1245ca18ffc06"}},{"code_sha256_prefix":"d365519671b5f160","entry":"mark_adapter_layernorm_cls_trainable","repo":"yifanycc/adazeta","repo_kind":"official","path":"large_models/loretta/tuners/loretta_adp.py","file_url":"https://github.com/yifanycc/adazeta/blob/HEAD/large_models/loretta/tuners/loretta_adp.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"d365519671b5f160"}},{"code_sha256_prefix":"2c40655921d4f49f","entry":"wrapped_linear_layers","repo":"yifanycc/adazeta","repo_kind":"official","path":"large_models/loretta/tuners/loretta_adp.py","file_url":"https://github.com/yifanycc/adazeta/blob/HEAD/large_models/loretta/tuners/loretta_adp.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"GPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"2c40655921d4f49f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}