{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/gemstones-a-model-suite-for-multi-faceted","title":"Gemstones: A Model Suite for Multi-Faceted Scaling Laws","arxiv_id":"2502.06857","date":"2025-02-07","proceeding":null,"authors":["Sean McLeish","John Kirchenbauer","David Yu Miller","Siddharth Singh","Abhinav Bhatele","Micah Goldblum","Ashwinee Panda","Tom Goldstein"],"abstract":"Scaling laws are typically fit using a family of models with a narrow range of frozen hyper-parameter choices. In this work we study scaling laws using a wide range of architecture and hyper-parameter choices, and highlight their impact on resulting prescriptions. As a primary artifact of our research, we release the Gemstones: the most comprehensive open-source scaling law dataset to date, consisting of over 4000 checkpoints from transformers with up to 2 billion parameters; these models have been trained with different learning rates, cooldown schedules, and architectural shapes. Our checkpoints enable more complex studies of scaling, such as a law that predicts language modeling performance as a function of model width and depth. By examining the various facets of our model suite, we find that the prescriptions of scaling laws can be highly sensitive to the experimental design process and the specific model checkpoints used during fitting. Code: https://github.com/mcleish7/gemstone-scaling-laws","url_abs":"https://arxiv.org/abs/2502.06857v1","url_pdf":"https://arxiv.org/pdf/2502.06857v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"gemstones-a-model-suite-for-multi-faceted","repo_url":"https://github.com/mcleish7/gemstone-scaling-laws","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":null}],"tasks":[{"task_slug":"experimental-design","task_name":"Experimental Design"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2502.06857","atlas_url":"https://app.syntology.ai/?focus=2502.06857","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2502.06857"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/mcleish7/gemstone-scaling-laws","reach":null}],"summary":{"ran_fixture":3,"ran_draft_wrong":2,"ran_honours":1,"unverified":1},"by_repo_kind":{"official":{"samples":7,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"cebb926b740c77a3","entry":"get_bucket_minimizers","repo":"mcleish7/gemstone-scaling-laws","repo_kind":"official","path":"plotters/approach_1.py","file_url":"https://github.com/mcleish7/gemstone-scaling-laws/blob/HEAD/plotters/approach_1.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cebb926b740c77a3"}},{"code_sha256_prefix":"deade5da77dad0be","entry":"get_resource_hull","repo":"mcleish7/gemstone-scaling-laws","repo_kind":"official","path":"plotters/approach_1.py","file_url":"https://github.com/mcleish7/gemstone-scaling-laws/blob/HEAD/plotters/approach_1.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"deade5da77dad0be"}},{"code_sha256_prefix":"0d6bdb92627ee4ba","entry":"huber_loss_objective","repo":"mcleish7/gemstone-scaling-laws","repo_kind":"official","path":"depth_width.py","file_url":"https://github.com/mcleish7/gemstone-scaling-laws/blob/HEAD/depth_width.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0d6bdb92627ee4ba"}},{"code_sha256_prefix":"4debd1716c9dd4ce","entry":"optimize_params","repo":"mcleish7/gemstone-scaling-laws","repo_kind":"official","path":"depth_width.py","file_url":"https://github.com/mcleish7/gemstone-scaling-laws/blob/HEAD/depth_width.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4debd1716c9dd4ce"}},{"code_sha256_prefix":"9735dca15dccf3d4","entry":"plot_line_of_best_fit","repo":"mcleish7/gemstone-scaling-laws","repo_kind":"official","path":"plotters/approach_1.py","file_url":"https://github.com/mcleish7/gemstone-scaling-laws/blob/HEAD/plotters/approach_1.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9735dca15dccf3d4"}},{"code_sha256_prefix":"5918a3644522e837","entry":"round_sig","repo":"mcleish7/gemstone-scaling-laws","repo_kind":"official","path":"benchmarks/fineweb_loss_plot.py","file_url":"https://github.com/mcleish7/gemstone-scaling-laws/blob/HEAD/benchmarks/fineweb_loss_plot.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5918a3644522e837"}},{"code_sha256_prefix":"841cecfa49f3e52a","entry":"log_pred","repo":"mcleish7/gemstone-scaling-laws","repo_kind":"official","path":"depth_width.py","file_url":"https://github.com/mcleish7/gemstone-scaling-laws/blob/HEAD/depth_width.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"841cecfa49f3e52a"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}