{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/quantifying-degeneracy-in-singular-models-via","title":"The Local Learning Coefficient: A Singularity-Aware Complexity Measure","arxiv_id":"2308.12108","date":"2023-08-23","proceeding":null,"authors":["Edmund Lau","Zach Furman","George Wang","Daniel Murfet","Susan Wei"],"abstract":"The Local Learning Coefficient (LLC) is introduced as a novel complexity measure for deep neural networks (DNNs). Recognizing the limitations of traditional complexity measures, the LLC leverages Singular Learning Theory (SLT), which has long recognized the significance of singularities in the loss landscape geometry. This paper provides an extensive exploration of the LLC's theoretical underpinnings, offering both a clear definition and intuitive insights into its application. Moreover, we propose a new scalable estimator for the LLC, which is then effectively applied across diverse architectures including deep linear networks up to 100M parameters, ResNet image models, and transformer language models. Empirical evidence suggests that the LLC provides valuable insights into how training heuristics might influence the effective complexity of DNNs. Ultimately, the LLC emerges as a crucial tool for reconciling the apparent contradiction between deep learning's complexity and the principle of parsimony.","url_abs":"https://arxiv.org/abs/2308.12108v2","url_pdf":"https://arxiv.org/pdf/2308.12108v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"quantifying-degeneracy-in-singular-models-via","repo_url":"https://github.com/edmundlth/scalable_learning_coefficient_with_sgld","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"jax","reach":{"status":"ok"}},{"paper_slug":"quantifying-degeneracy-in-singular-models-via","repo_url":"https://github.com/nrimsky/devinterp","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"inductive-bias","task_name":"Inductive Bias"},{"task_slug":"learning-theory","task_name":"Learning Theory"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2308.12108","atlas_url":"https://app.syntology.ai/?focus=2308.12108","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2308.12108"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/nrimsky/devinterp","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/edmundlth/scalable_learning_coefficient_with_sgld","reach":{"status":"ok"}}],"summary":{"ran":7,"unverified":1},"by_repo_kind":{"official":{"samples":8,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":8,"samples":[{"code_sha256_prefix":"5f651b2ecf6a0472","entry":"SGLD_step","repo":"edmundlth/scalable_learning_coefficient_with_sgld","repo_kind":"official","path":"toy_potential.py","file_url":"https://github.com/edmundlth/scalable_learning_coefficient_with_sgld/blob/HEAD/toy_potential.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5f651b2ecf6a0472"}},{"code_sha256_prefix":"94c7c1497876ae6b","entry":"compute_bayesian_loss","repo":"edmundlth/scalable_learning_coefficient_with_sgld","repo_kind":"official","path":"bayes_observables.py","file_url":"https://github.com/edmundlth/scalable_learning_coefficient_with_sgld/blob/HEAD/bayes_observables.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"94c7c1497876ae6b"}},{"code_sha256_prefix":"e12f448719610926","entry":"compute_gibbs_loss","repo":"edmundlth/scalable_learning_coefficient_with_sgld","repo_kind":"official","path":"bayes_observables.py","file_url":"https://github.com/edmundlth/scalable_learning_coefficient_with_sgld/blob/HEAD/bayes_observables.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e12f448719610926"}},{"code_sha256_prefix":"4302d56b70cb4602","entry":"compute_gibbs_loss_vectorised","repo":"edmundlth/scalable_learning_coefficient_with_sgld","repo_kind":"official","path":"bayes_observables.py","file_url":"https://github.com/edmundlth/scalable_learning_coefficient_with_sgld/blob/HEAD/bayes_observables.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4302d56b70cb4602"}},{"code_sha256_prefix":"84f0c2e7a96da300","entry":"expand_dictionary","repo":"edmundlth/scalable_learning_coefficient_with_sgld","repo_kind":"official","path":"utils.py","file_url":"https://github.com/edmundlth/scalable_learning_coefficient_with_sgld/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"84f0c2e7a96da300"}},{"code_sha256_prefix":"fd85d6a7dda2532b","entry":"process_json_file","repo":"edmundlth/scalable_learning_coefficient_with_sgld","repo_kind":"official","path":"toy_potential.py","file_url":"https://github.com/edmundlth/scalable_learning_coefficient_with_sgld/blob/HEAD/toy_potential.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fd85d6a7dda2532b"}},{"code_sha256_prefix":"d95418e31eb2b4a0","entry":"run_SGLD","repo":"edmundlth/scalable_learning_coefficient_with_sgld","repo_kind":"official","path":"toy_potential.py","file_url":"https://github.com/edmundlth/scalable_learning_coefficient_with_sgld/blob/HEAD/toy_potential.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d95418e31eb2b4a0"}},{"code_sha256_prefix":"2e307aa1305a8f05","entry":"create_gaussian_noise_regression_dataset","repo":"edmundlth/scalable_learning_coefficient_with_sgld","repo_kind":"official","path":"model.py","file_url":"https://github.com/edmundlth/scalable_learning_coefficient_with_sgld/blob/HEAD/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2e307aa1305a8f05"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}