{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/fantastic-weights-and-how-to-find-them-where-1","title":"Fantastic Weights and How to Find Them: Where to Prune in Dynamic Sparse Training","arxiv_id":"2306.12230","date":"2023-06-21","proceeding":"NeurIPS 2023 11","authors":["Aleksandra I. Nowak","Bram Grooten","Decebal Constantin Mocanu","Jacek Tabor"],"abstract":"Dynamic Sparse Training (DST) is a rapidly evolving area of research that seeks to optimize the sparse initialization of a neural network by adapting its topology during training. It has been shown that under specific conditions, DST is able to outperform dense models. The key components of this framework are the pruning and growing criteria, which are repeatedly applied during the training process to adjust the network's sparse connectivity. While the growing criterion's impact on DST performance is relatively well studied, the influence of the pruning criterion remains overlooked. To address this issue, we design and perform an extensive empirical analysis of various pruning criteria to better understand their impact on the dynamics of DST solutions. Surprisingly, we find that most of the studied methods yield similar results. The differences become more significant in the low-density regime, where the best performance is predominantly given by the simplest technique: magnitude-based pruning. The code is provided at https://github.com/alooow/fantastic_weights_paper","url_abs":"https://arxiv.org/abs/2306.12230v2","url_pdf":"https://arxiv.org/pdf/2306.12230v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"fantastic-weights-and-how-to-find-them-where-1","repo_url":"https://github.com/alooow/fantastic_weights_paper","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[],"methods":[{"method_slug":"dst","method_name":"DST"},{"method_slug":"pruning","method_name":"Pruning"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2306.12230","atlas_url":"https://app.syntology.ai/?focus=2306.12230","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.12230"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/alooow/fantastic_weights_paper","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/TimDettmers/sparse_learning","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":2,"unverified":18},"by_repo_kind":{"official":{"samples":12,"ran":2,"repositories":1},"found_in_text":{"samples":8,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"12d59cf9ada8b41a","entry":"snip_forward_conv2d","repo":"alooow/fantastic_weights_paper","repo_kind":"official","path":"ImageNet/snip.py","file_url":"https://github.com/alooow/fantastic_weights_paper/blob/HEAD/ImageNet/snip.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"12d59cf9ada8b41a"}},{"code_sha256_prefix":"94eeaaa88bd57941","entry":"snip_forward_linear","repo":"alooow/fantastic_weights_paper","repo_kind":"official","path":"ImageNet/snip.py","file_url":"https://github.com/alooow/fantastic_weights_paper/blob/HEAD/ImageNet/snip.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"94eeaaa88bd57941"}},{"code_sha256_prefix":"04768d98372daca0","entry":"SNIP","repo":"alooow/fantastic_weights_paper","repo_kind":"official","path":"ImageNet/snip.py","file_url":"https://github.com/alooow/fantastic_weights_paper/blob/HEAD/ImageNet/snip.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"04768d98372daca0"}},{"code_sha256_prefix":"51a6a5c2a049e900","entry":"build_resnet","repo":"TimDettmers/sparse_learning","repo_kind":"found_in_text","path":"imagenet/tuned_resnet/resnet.py","file_url":"https://github.com/TimDettmers/sparse_learning/blob/HEAD/imagenet/tuned_resnet/resnet.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"51a6a5c2a049e900"}},{"code_sha256_prefix":"f069023ee573fe51","entry":"build_resnet","repo":"alooow/fantastic_weights_paper","repo_kind":"official","path":"ImageNet/resnet.py","file_url":"https://github.com/alooow/fantastic_weights_paper/blob/HEAD/ImageNet/resnet.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f069023ee573fe51"}},{"code_sha256_prefix":"61bbac980274b60d","entry":"build_resnet","repo":"alooow/fantastic_weights_paper","repo_kind":"official","path":"models/imagenet_resnet.py","file_url":"https://github.com/alooow/fantastic_weights_paper/blob/HEAD/models/imagenet_resnet.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"61bbac980274b60d"}},{"code_sha256_prefix":"20c4b599af53b4ed","entry":"diverging_colors","repo":"TimDettmers/sparse_learning","repo_kind":"found_in_text","path":"plot_graphs.py","file_url":"https://github.com/TimDettmers/sparse_learning/blob/HEAD/plot_graphs.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"20c4b599af53b4ed"}},{"code_sha256_prefix":"717c76ec787ddfe7","entry":"fast_collate","repo":"alooow/fantastic_weights_paper","repo_kind":"official","path":"imagenet_main.py","file_url":"https://github.com/alooow/fantastic_weights_paper/blob/HEAD/imagenet_main.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"717c76ec787ddfe7"}},{"code_sha256_prefix":"410b25b6d0e7a081","entry":"get_optimizer","repo":"alooow/fantastic_weights_paper","repo_kind":"official","path":"imagenet_main.py","file_url":"https://github.com/alooow/fantastic_weights_paper/blob/HEAD/imagenet_main.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"410b25b6d0e7a081"}},{"code_sha256_prefix":"04b0a36942f99acf","entry":"get_train_loader","repo":"alooow/fantastic_weights_paper","repo_kind":"official","path":"imagenet_main.py","file_url":"https://github.com/alooow/fantastic_weights_paper/blob/HEAD/imagenet_main.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"04b0a36942f99acf"}},{"code_sha256_prefix":"39e82a7c002940de","entry":"initializations","repo":"alooow/fantastic_weights_paper","repo_kind":"official","path":"models/initializers.py","file_url":"https://github.com/alooow/fantastic_weights_paper/blob/HEAD/models/initializers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"39e82a7c002940de"}},{"code_sha256_prefix":"7fbf3f0c2be8ea47","entry":"magnitude_redistribution","repo":"alooow/fantastic_weights_paper","repo_kind":"official","path":"ImageNet/funcs.py","file_url":"https://github.com/alooow/fantastic_weights_paper/blob/HEAD/ImageNet/funcs.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7fbf3f0c2be8ea47"}},{"code_sha256_prefix":"85588244473b88f9","entry":"momentum_redistribution","repo":"alooow/fantastic_weights_paper","repo_kind":"official","path":"ImageNet/funcs.py","file_url":"https://github.com/alooow/fantastic_weights_paper/blob/HEAD/ImageNet/funcs.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"85588244473b88f9"}},{"code_sha256_prefix":"e67bbb994ab5bae5","entry":"nonzero_redistribution","repo":"alooow/fantastic_weights_paper","repo_kind":"official","path":"ImageNet/funcs.py","file_url":"https://github.com/alooow/fantastic_weights_paper/blob/HEAD/ImageNet/funcs.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e67bbb994ab5bae5"}},{"code_sha256_prefix":"b8a31ec73b7a8b3c","entry":"rainbow_colors","repo":"TimDettmers/sparse_learning","repo_kind":"found_in_text","path":"plot_graphs.py","file_url":"https://github.com/TimDettmers/sparse_learning/blob/HEAD/plot_graphs.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b8a31ec73b7a8b3c"}},{"code_sha256_prefix":"4746465c14058f8a","entry":"sequential_colors","repo":"TimDettmers/sparse_learning","repo_kind":"found_in_text","path":"plot_graphs.py","file_url":"https://github.com/TimDettmers/sparse_learning/blob/HEAD/plot_graphs.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4746465c14058f8a"}},{"code_sha256_prefix":"bb11b6868648aca9","entry":"uniform_coverage","repo":"TimDettmers/sparse_learning","repo_kind":"found_in_text","path":"imagenet/baseline/parameterized_tensors.py","file_url":"https://github.com/TimDettmers/sparse_learning/blob/HEAD/imagenet/baseline/parameterized_tensors.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bb11b6868648aca9"}},{"code_sha256_prefix":"177842ec69eed409","entry":"variance_redistribution","repo":"TimDettmers/sparse_learning","repo_kind":"found_in_text","path":"mnist_cifar/extensions.py","file_url":"https://github.com/TimDettmers/sparse_learning/blob/HEAD/mnist_cifar/extensions.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"177842ec69eed409"}},{"code_sha256_prefix":"77b9016770359ac8","entry":"your_pruning","repo":"TimDettmers/sparse_learning","repo_kind":"found_in_text","path":"mnist_cifar/extensions.py","file_url":"https://github.com/TimDettmers/sparse_learning/blob/HEAD/mnist_cifar/extensions.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"77b9016770359ac8"}},{"code_sha256_prefix":"37199b84c6e58a0e","entry":"your_redistribution","repo":"TimDettmers/sparse_learning","repo_kind":"found_in_text","path":"mnist_cifar/extensions.py","file_url":"https://github.com/TimDettmers/sparse_learning/blob/HEAD/mnist_cifar/extensions.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"37199b84c6e58a0e"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}