{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/oats-outlier-aware-pruning-through-sparse-and","title":"OATS: Outlier-Aware Pruning Through Sparse and Low Rank Decomposition","arxiv_id":"2409.13652","date":"2024-09-20","proceeding":null,"authors":["Stephen Zhang","Vardan Papyan"],"abstract":"The recent paradigm shift to large-scale foundation models has brought about a new era for deep learning that, while has found great success in practice, has also been plagued by prohibitively expensive costs in terms of high memory consumption and compute. To mitigate these issues, there has been a concerted effort in post-hoc neural network pruning techniques that do not require costly retraining. Despite the considerable progress being made, existing methods often exhibit a steady drop in model performance as the compression increases. In this paper, we present a novel approach to compressing large transformers, coined OATS, that utilizes the second moment information in the input embeddings to decompose the model weights into a sum of sparse and low-rank matrices. Without any retraining, OATS achieves state-of-the-art performance when compressing models by up to $60\\%$ on large language models such as Llama-3 and Phi-3 and vision transformers such as ViT and DINOv2 while delivering up to $1.37\\times$ the CPU acceleration versus a model that was comparably pruned.","url_abs":"https://arxiv.org/abs/2409.13652v3","url_pdf":"https://arxiv.org/pdf/2409.13652v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"oats-outlier-aware-pruning-through-sparse-and","repo_url":"https://github.com/stephenqz/oats","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":null,"task_name":"CPU"},{"task_slug":"network-pruning","task_name":"Network Pruning"}],"methods":[{"method_slug":"pruning","method_name":"Pruning"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2409.13652","atlas_url":"https://app.syntology.ai/?focus=2409.13652","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2409.13652"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/stephenqz/oats","reach":{"status":"ok"}}],"summary":{"ran_draft_wrong":1,"ran":4,"unverified":1},"by_repo_kind":{"official":{"samples":6,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":6,"samples":[{"code_sha256_prefix":"14fab8e91e876c60","entry":"find_layers","repo":"stephenqz/oats","repo_kind":"official","path":"OATS/pruning_utils.py","file_url":"https://github.com/stephenqz/oats/blob/HEAD/OATS/pruning_utils.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"14fab8e91e876c60"}},{"code_sha256_prefix":"62216694a6019cdb","entry":"get_layer0_inputs","repo":"stephenqz/oats","repo_kind":"official","path":"vit/prune_oats_vit.py","file_url":"https://github.com/stephenqz/oats/blob/HEAD/vit/prune_oats_vit.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"62216694a6019cdb"}},{"code_sha256_prefix":"660047a46e099ddf","entry":"load_checkpoint","repo":"stephenqz/oats","repo_kind":"official","path":"OATS/pruning_utils.py","file_url":"https://github.com/stephenqz/oats/blob/HEAD/OATS/pruning_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"660047a46e099ddf"}},{"code_sha256_prefix":"aa35d790b60452f7","entry":"map_tensors","repo":"stephenqz/oats","repo_kind":"official","path":"gpu_utils.py","file_url":"https://github.com/stephenqz/oats/blob/HEAD/gpu_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"aa35d790b60452f7"}},{"code_sha256_prefix":"b8bb7d55280d7e8f","entry":"prepare_dataloader","repo":"stephenqz/oats","repo_kind":"official","path":"data_utils.py","file_url":"https://github.com/stephenqz/oats/blob/HEAD/data_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b8bb7d55280d7e8f"}},{"code_sha256_prefix":"8e17212c620812e4","entry":"get_dataset","repo":"stephenqz/oats","repo_kind":"official","path":"data_utils.py","file_url":"https://github.com/stephenqz/oats/blob/HEAD/data_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8e17212c620812e4"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}