{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/two-sparse-matrices-are-better-than-one","title":"Two Sparse Matrices are Better than One: Sparsifying Neural Networks with Double Sparse Factorization","arxiv_id":"2409.18850","date":"2024-09-27","proceeding":null,"authors":["Vladimír Boža","Vladimír Macko"],"abstract":"Neural networks are often challenging to work with due to their large size and complexity. To address this, various methods aim to reduce model size by sparsifying or decomposing weight matrices, such as magnitude pruning and low-rank or block-diagonal factorization. In this work, we present Double Sparse Factorization (DSF), where we factorize each weight matrix into two sparse matrices. Although solving this problem exactly is computationally infeasible, we propose an efficient heuristic based on alternating minimization via ADMM that achieves state-of-the-art results, enabling unprecedented sparsification of neural networks. For instance, in a one-shot pruning setting, our method can reduce the size of the LLaMA2-13B model by 50% while maintaining better performance than the dense LLaMA2-7B model. We also compare favorably with Optimal Brain Compression, the state-of-the-art layer-wise pruning approach for convolutional neural networks. Furthermore, accuracy improvements of our method persist even after further model fine-tuning. Code available at: https://github.com/usamec/double_sparse.","url_abs":"https://arxiv.org/abs/2409.18850v1","url_pdf":"https://arxiv.org/pdf/2409.18850v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"two-sparse-matrices-are-better-than-one","repo_url":"https://github.com/usamec/double_sparse","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[],"methods":[{"method_slug":"admm","method_name":"ADMM"},{"method_slug":"pruning","method_name":"Pruning"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2409.18850","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2409.18850"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/usamec/double_sparse","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":5,"ran_draft_wrong":1,"unverified":6},"by_repo_kind":{"official":{"samples":12,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"49a001a61f2633ae","entry":"ent","repo":"usamec/double_sparse","repo_kind":"official","path":"doublesparse.py","file_url":"https://github.com/usamec/double_sparse/blob/HEAD/doublesparse.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"49a001a61f2633ae"}},{"code_sha256_prefix":"a9e7f2cdf016b88b","entry":"find_layers","repo":"usamec/double_sparse","repo_kind":"official","path":"modelutils.py","file_url":"https://github.com/usamec/double_sparse/blob/HEAD/modelutils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a9e7f2cdf016b88b"}},{"code_sha256_prefix":"2c35e419392e5ae7","entry":"find_other2","repo":"usamec/double_sparse","repo_kind":"official","path":"doublesparse.py","file_url":"https://github.com/usamec/double_sparse/blob/HEAD/doublesparse.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2c35e419392e5ae7"}},{"code_sha256_prefix":"0723e2735373f4d8","entry":"get_imagenet","repo":"usamec/double_sparse","repo_kind":"official","path":"other_experiments/datautils.py","file_url":"https://github.com/usamec/double_sparse/blob/HEAD/other_experiments/datautils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0723e2735373f4d8"}},{"code_sha256_prefix":"bf5b0ce87ee19903","entry":"get_llama","repo":"usamec/double_sparse","repo_kind":"official","path":"llama.py","file_url":"https://github.com/usamec/double_sparse/blob/HEAD/llama.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bf5b0ce87ee19903"}},{"code_sha256_prefix":"a1c5a6a697a88d18","entry":"mag_prune","repo":"usamec/double_sparse","repo_kind":"official","path":"doublesparse.py","file_url":"https://github.com/usamec/double_sparse/blob/HEAD/doublesparse.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a1c5a6a697a88d18"}},{"code_sha256_prefix":"75dbb95badb47b1a","entry":"get_coco","repo":"usamec/double_sparse","repo_kind":"official","path":"other_experiments/datautils.py","file_url":"https://github.com/usamec/double_sparse/blob/HEAD/other_experiments/datautils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"75dbb95badb47b1a"}},{"code_sha256_prefix":"a7cbf4fc49c71e03","entry":"get_ptb","repo":"usamec/double_sparse","repo_kind":"official","path":"datautils.py","file_url":"https://github.com/usamec/double_sparse/blob/HEAD/datautils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a7cbf4fc49c71e03"}},{"code_sha256_prefix":"729f957048af4a3f","entry":"get_tokenizer","repo":"usamec/double_sparse","repo_kind":"official","path":"datautils.py","file_url":"https://github.com/usamec/double_sparse/blob/HEAD/datautils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"729f957048af4a3f"}},{"code_sha256_prefix":"011909315dfdbc2e","entry":"get_wikitext2","repo":"usamec/double_sparse","repo_kind":"official","path":"datautils.py","file_url":"https://github.com/usamec/double_sparse/blob/HEAD/datautils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"011909315dfdbc2e"}},{"code_sha256_prefix":"14961351f5344dc1","entry":"llama_sequential","repo":"usamec/double_sparse","repo_kind":"official","path":"llama.py","file_url":"https://github.com/usamec/double_sparse/blob/HEAD/llama.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"14961351f5344dc1"}},{"code_sha256_prefix":"52ba3cdd84734157","entry":"random_subset","repo":"usamec/double_sparse","repo_kind":"official","path":"other_experiments/datautils.py","file_url":"https://github.com/usamec/double_sparse/blob/HEAD/other_experiments/datautils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"52ba3cdd84734157"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}