{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/measuring-sharpness-in-grokking","title":"Measuring Sharpness in Grokking","arxiv_id":"2402.08946","date":"2024-02-14","proceeding":null,"authors":["Jack Miller","Patrick Gleeson","Charles O'Neill","Thang Bui","Noam Levi"],"abstract":"Neural networks sometimes exhibit grokking, a phenomenon where perfect or near-perfect performance is achieved on a validation set well after the same performance has been obtained on the corresponding training set. In this workshop paper, we introduce a robust technique for measuring grokking, based on fitting an appropriate functional form. We then use this to investigate the sharpness of transitions in training and validation accuracy under two settings. The first setting is the theoretical framework developed by Levi et al. (2023) where closed form expressions are readily accessible. The second setting is a two-layer MLP trained to predict the parity of bits, with grokking induced by the concealment strategy of Miller et al. (2023). We find that trends between relative grokking gap and grokking sharpness are similar in both settings when using absolute and relative measures of sharpness. Reflecting on this, we make progress toward explaining some trends and identify the need for further study to untangle the various mechanisms which influence the sharpness of grokking.","url_abs":"https://arxiv.org/abs/2402.08946v1","url_pdf":"https://arxiv.org/pdf/2402.08946v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"measuring-sharpness-in-grokking","repo_url":"https://github.com/jackmiller2003/tiny-gen","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[],"methods":[{"method_slug":"set","method_name":"SET"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2402.08946","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.08946"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jackmiller2003/tiny-gen","reach":{"status":"ok"}}],"summary":{"ran":6,"unverified":1},"by_repo_kind":{"official":{"samples":7,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":7,"samples":[{"code_sha256_prefix":"34f259f02d628741","entry":"add_features_for_lr_classification","repo":"jackmiller2003/tiny-gen","repo_kind":"official","path":"tools.py","file_url":"https://github.com/jackmiller2003/tiny-gen/blob/HEAD/tools.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"34f259f02d628741"}},{"code_sha256_prefix":"e9758b9ed9724a62","entry":"combine_datasets","repo":"jackmiller2003/tiny-gen","repo_kind":"official","path":"src/dataset.py","file_url":"https://github.com/jackmiller2003/tiny-gen/blob/HEAD/src/dataset.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e9758b9ed9724a62"}},{"code_sha256_prefix":"f2e62af61a2171aa","entry":"gaussian_loss","repo":"jackmiller2003/tiny-gen","repo_kind":"official","path":"tools.py","file_url":"https://github.com/jackmiller2003/tiny-gen/blob/HEAD/tools.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f2e62af61a2171aa"}},{"code_sha256_prefix":"1239ce880d8bb156","entry":"generate_zero_one_classification","repo":"jackmiller2003/tiny-gen","repo_kind":"official","path":"src/dataset.py","file_url":"https://github.com/jackmiller2003/tiny-gen/blob/HEAD/src/dataset.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1239ce880d8bb156"}},{"code_sha256_prefix":"4ddde33fb0d7122b","entry":"get_accuracy","repo":"jackmiller2003/tiny-gen","repo_kind":"official","path":"src/common.py","file_url":"https://github.com/jackmiller2003/tiny-gen/blob/HEAD/src/common.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4ddde33fb0d7122b"}},{"code_sha256_prefix":"e58241cc46e5cd07","entry":"l2_norm_for_lr","repo":"jackmiller2003/tiny-gen","repo_kind":"official","path":"tools.py","file_url":"https://github.com/jackmiller2003/tiny-gen/blob/HEAD/tools.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e58241cc46e5cd07"}},{"code_sha256_prefix":"5f46d991c13efaed","entry":"mvn_log_prob","repo":"jackmiller2003/tiny-gen","repo_kind":"official","path":"src/model.py","file_url":"https://github.com/jackmiller2003/tiny-gen/blob/HEAD/src/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5f46d991c13efaed"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}