{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/gradient-free-adaptive-global-pruning-for-pre","title":"SparseLLM: Towards Global Pruning for Pre-trained Language Models","arxiv_id":"2402.17946","date":"2024-02-28","proceeding":null,"authors":["Guangji Bai","Yijiang Li","Chen Ling","Kibaek Kim","Liang Zhao"],"abstract":"The transformative impact of large language models (LLMs) like LLaMA and GPT on natural language processing is countered by their prohibitive computational demands. Pruning has emerged as a pivotal compression strategy, introducing sparsity to enhance both memory and computational efficiency. Yet, traditional global pruning is impractical for LLMs due to scalability issues, while local pruning, despite its efficiency, leads to suboptimal solutions. Addressing these challenges, we propose SparseLLM, a novel framework that redefines the global pruning process into manageable, coordinated subproblems, allowing for resource-efficient optimization with global optimality. SparseLLM's approach, which conceptualizes LLMs as a chain of modular functions and leverages auxiliary variables for problem decomposition, not only facilitates a pragmatic application on LLMs but also demonstrates significant performance improvements, particularly in high-sparsity regimes where it surpasses current state-of-the-art methods.","url_abs":"https://arxiv.org/abs/2402.17946v4","url_pdf":"https://arxiv.org/pdf/2402.17946v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"gradient-free-adaptive-global-pruning-for-pre","repo_url":"https://github.com/baithebest/adagp","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"gradient-free-adaptive-global-pruning-for-pre","repo_url":"https://github.com/baithebest/sparsellm","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"computational-efficiency","task_name":"Computational Efficiency"},{"task_slug":"problem-decomposition","task_name":"Problem Decomposition"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"cosine-annealing","method_name":"Cosine Annealing"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"discriminative-fine-tuning","method_name":"Discriminative Fine-Tuning"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt","method_name":"GPT"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-cosine-annealing","method_name":"Linear Warmup With Cosine Annealing"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"pruning","method_name":"Pruning"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2402.17946","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.17946"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/BaiTheBest/SparseLLM","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/baithebest/sparsellm","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/baithebest/adagp","reach":null}],"summary":{"ran":2,"ran_draft_wrong":1,"unverified":6},"by_repo_kind":{"official":{"samples":9,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"b5fc92a2899d6d8a","entry":"SparseGPT_OPT","repo":"baithebest/sparsellm","repo_kind":"official","path":"pruning_utils.py","file_url":"https://github.com/baithebest/sparsellm/blob/HEAD/pruning_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b5fc92a2899d6d8a"}},{"code_sha256_prefix":"a9e7f2cdf016b88b","entry":"find_layers","repo":"BaiTheBest/SparseLLM","repo_kind":"official","path":"pruning_utils.py","file_url":"https://github.com/BaiTheBest/SparseLLM/blob/HEAD/pruning_utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a9e7f2cdf016b88b"}},{"code_sha256_prefix":"50ceff9d34d96d60","entry":"quantize","repo":"BaiTheBest/SparseLLM","repo_kind":"official","path":"quant.py","file_url":"https://github.com/BaiTheBest/SparseLLM/blob/HEAD/quant.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"50ceff9d34d96d60"}},{"code_sha256_prefix":"0ac474c6b7daab48","entry":"get_llama","repo":"BaiTheBest/SparseLLM","repo_kind":"official","path":"model_utils.py","file_url":"https://github.com/BaiTheBest/SparseLLM/blob/HEAD/model_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0ac474c6b7daab48"}},{"code_sha256_prefix":"ca396f6562e13af3","entry":"get_opt","repo":"BaiTheBest/SparseLLM","repo_kind":"official","path":"model_utils.py","file_url":"https://github.com/BaiTheBest/SparseLLM/blob/HEAD/model_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ca396f6562e13af3"}},{"code_sha256_prefix":"a7cbf4fc49c71e03","entry":"get_ptb","repo":"BaiTheBest/SparseLLM","repo_kind":"official","path":"datautils.py","file_url":"https://github.com/BaiTheBest/SparseLLM/blob/HEAD/datautils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a7cbf4fc49c71e03"}},{"code_sha256_prefix":"729f957048af4a3f","entry":"get_tokenizer","repo":"BaiTheBest/SparseLLM","repo_kind":"official","path":"datautils.py","file_url":"https://github.com/BaiTheBest/SparseLLM/blob/HEAD/datautils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"729f957048af4a3f"}},{"code_sha256_prefix":"011909315dfdbc2e","entry":"get_wikitext2","repo":"BaiTheBest/SparseLLM","repo_kind":"official","path":"datautils.py","file_url":"https://github.com/BaiTheBest/SparseLLM/blob/HEAD/datautils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"011909315dfdbc2e"}},{"code_sha256_prefix":"dd6dc15886074ef8","entry":"opt_sparsellm","repo":"BaiTheBest/SparseLLM","repo_kind":"official","path":"model_utils.py","file_url":"https://github.com/BaiTheBest/SparseLLM/blob/HEAD/model_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"dd6dc15886074ef8"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}