{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/towards-scalable-and-versatile-weight-space","title":"Towards Scalable and Versatile Weight Space Learning","arxiv_id":"2406.09997","date":"2024-06-14","proceeding":null,"authors":["Konstantin Schürholt","Michael W. Mahoney","Damian Borth"],"abstract":"Learning representations of well-trained neural network models holds the promise to provide an understanding of the inner workings of those models. However, previous work has either faced limitations when processing larger networks or was task-specific to either discriminative or generative tasks. This paper introduces the SANE approach to weight-space learning. SANE overcomes previous limitations by learning task-agnostic representations of neural networks that are scalable to larger models of varying architectures and that show capabilities beyond a single task. Our method extends the idea of hyper-representations towards sequential processing of subsets of neural network weights, thus allowing one to embed larger neural networks as a set of tokens into the learned representation space. SANE reveals global model information from layer-wise embeddings, and it can sequentially generate unseen neural network models, which was unattainable with previous hyper-representation learning methods. Extensive empirical evaluation demonstrates that SANE matches or exceeds state-of-the-art performance on several weight representation learning benchmarks, particularly in initialization for new tasks and larger ResNet architectures.","url_abs":"https://arxiv.org/abs/2406.09997v1","url_pdf":"https://arxiv.org/pdf/2406.09997v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"towards-scalable-and-versatile-weight-space","repo_url":"https://github.com/hsg-aiml/sane","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"representation-learning","task_name":"Representation Learning"},{"task_slug":"weight-space-learning","task_name":"Weight Space Learning"}],"methods":[{"method_slug":"average-pooling","method_name":"Average Pooling"},{"method_slug":"convolution","method_name":"Convolution"},{"method_slug":"global-average-pooling","method_name":"Global Average Pooling"},{"method_slug":"kaiming-initialization","method_name":"Kaiming Initialization"},{"method_slug":"max-pooling","method_name":"Max Pooling"},{"method_slug":"set","method_name":"SET"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2406.09997","atlas_url":"https://app.syntology.ai/?focus=2406.09997","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.09997"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/HSG-AIML/SANE","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/hsg-aiml/sane","reach":{"status":"ok"}}],"summary":{"ran":11,"unverified":4},"by_repo_kind":{"official":{"samples":15,"ran":11,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":15,"samples":[{"code_sha256_prefix":"e3b41e6df5019382","entry":"Block","repo":"hsg-aiml/sane","repo_kind":"official","path":"src/SANE/models/def_AE.py","file_url":"https://github.com/hsg-aiml/sane/blob/HEAD/src/SANE/models/def_AE.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e3b41e6df5019382"}},{"code_sha256_prefix":"62d9432b16366a2a","entry":"LayerNorm","repo":"hsg-aiml/sane","repo_kind":"official","path":"src/SANE/models/def_AE.py","file_url":"https://github.com/hsg-aiml/sane/blob/HEAD/src/SANE/models/def_AE.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"62d9432b16366a2a"}},{"code_sha256_prefix":"4b37e2f3f3e35907","entry":"MLP","repo":"hsg-aiml/sane","repo_kind":"official","path":"src/SANE/models/def_AE.py","file_url":"https://github.com/hsg-aiml/sane/blob/HEAD/src/SANE/models/def_AE.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4b37e2f3f3e35907"}},{"code_sha256_prefix":"56918c3afce71745","entry":"PositionEmbs","repo":"hsg-aiml/sane","repo_kind":"official","path":"src/SANE/models/def_AE.py","file_url":"https://github.com/hsg-aiml/sane/blob/HEAD/src/SANE/models/def_AE.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"56918c3afce71745"}},{"code_sha256_prefix":"e81937cece236ca0","entry":"SelfAttention","repo":"hsg-aiml/sane","repo_kind":"official","path":"src/SANE/models/def_AE.py","file_url":"https://github.com/hsg-aiml/sane/blob/HEAD/src/SANE/models/def_AE.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e81937cece236ca0"}},{"code_sha256_prefix":"10cc09608b9478b4","entry":"SimpleProjectionHead","repo":"hsg-aiml/sane","repo_kind":"official","path":"src/SANE/models/def_AE.py","file_url":"https://github.com/hsg-aiml/sane/blob/HEAD/src/SANE/models/def_AE.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"10cc09608b9478b4"}},{"code_sha256_prefix":"c2a9b982aaf5a3cd","entry":"TransformerEncoder","repo":"hsg-aiml/sane","repo_kind":"official","path":"src/SANE/models/def_AE.py","file_url":"https://github.com/hsg-aiml/sane/blob/HEAD/src/SANE/models/def_AE.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c2a9b982aaf5a3cd"}},{"code_sha256_prefix":"889fe8497b0ba7c8","entry":"permute_model_vector","repo":"HSG-AIML/SANE","repo_kind":"official","path":"src/SANE/datasets/augmentations.py","file_url":"https://github.com/HSG-AIML/SANE/blob/HEAD/src/SANE/datasets/augmentations.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"889fe8497b0ba7c8"}},{"code_sha256_prefix":"5ac745803cfc2796","entry":"read_properties_from_path","repo":"HSG-AIML/SANE","repo_kind":"official","path":"src/SANE/datasets/dataset_epochs.py","file_url":"https://github.com/HSG-AIML/SANE/blob/HEAD/src/SANE/datasets/dataset_epochs.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5ac745803cfc2796"}},{"code_sha256_prefix":"ec1ff9919f412754","entry":"test_checkpoint_for_nan","repo":"HSG-AIML/SANE","repo_kind":"official","path":"src/SANE/datasets/dataset_auxiliaries.py","file_url":"https://github.com/HSG-AIML/SANE/blob/HEAD/src/SANE/datasets/dataset_auxiliaries.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ec1ff9919f412754"}},{"code_sha256_prefix":"d2a9bd8273d8f5d4","entry":"test_checkpoint_with_threshold","repo":"HSG-AIML/SANE","repo_kind":"official","path":"src/SANE/datasets/dataset_auxiliaries.py","file_url":"https://github.com/HSG-AIML/SANE/blob/HEAD/src/SANE/datasets/dataset_auxiliaries.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d2a9bd8273d8f5d4"}},{"code_sha256_prefix":"e89a53ea969fce78","entry":"AE","repo":"hsg-aiml/sane","repo_kind":"official","path":"src/SANE/models/def_AE.py","file_url":"https://github.com/hsg-aiml/sane/blob/HEAD/src/SANE/models/def_AE.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e89a53ea969fce78"}},{"code_sha256_prefix":"31ce1adb9a55256c","entry":"get_net_epoch_lst_from_label","repo":"HSG-AIML/SANE","repo_kind":"official","path":"src/SANE/datasets/dataset_auxiliaries.py","file_url":"https://github.com/HSG-AIML/SANE/blob/HEAD/src/SANE/datasets/dataset_auxiliaries.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"31ce1adb9a55256c"}},{"code_sha256_prefix":"790ee3cda94b46c4","entry":"read_properties_from_path","repo":"HSG-AIML/SANE","repo_kind":"official","path":"src/SANE/datasets/dataset_properties.py","file_url":"https://github.com/HSG-AIML/SANE/blob/HEAD/src/SANE/datasets/dataset_properties.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"790ee3cda94b46c4"}},{"code_sha256_prefix":"20834a80f80685f1","entry":"sleep_then_increment","repo":"HSG-AIML/SANE","repo_kind":"official","path":"src/SANE/datasets/progress_bar.py","file_url":"https://github.com/HSG-AIML/SANE/blob/HEAD/src/SANE/datasets/progress_bar.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"20834a80f80685f1"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}