{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2510-24963","title":"Language Model Behavioral Phases are Consistent Across Architecture, Training Data, and Scale","arxiv_id":"2510.24963","date":"2025-10-28","proceeding":"NeurIPS","authors":["James A. Michaelov","Roger P. Levy","Benjamin K. Bergen"],"abstract":"We show that across architecture (Transformer vs. Mamba vs. RWKV), training dataset (OpenWebText vs. The Pile), and scale (14 million parameters to 12 billion parameters), autoregressive language models exhibit highly consistent patterns of change in their behavior over the course of pretraining. Based on our analysis of over 1,400 language model checkpoints on over 110,000 tokens of English, we find that up to 98% of the variance in language model behavior at the word level can be explained by three simple heuristics: the unigram probability (frequency) of a given word, the $n$-gram probability of the word, and the semantic similarity between the word and its context. Furthermore, we see consistent behavioral phases in all language models, with their predicted probabilities for words overfitting to those words' $n$-gram probabilities for increasing $n$ over the course of training. Taken together, these results suggest that learning in neural language models may follow a similar trajectory irrespective of model details.","url_abs":"https://arxiv.org/abs/2510.24963","url_pdf":"https://arxiv.org/pdf/2510.24963","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2510.24963","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2510.24963"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/jmichaelov/lm-behavioral-phases","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"unverified":5},"by_repo_kind":{"found_in_text":{"samples":5,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"4ab3a7754330400c","entry":"get_metric_name","repo":"jmichaelov/lm-behavioral-phases","repo_kind":"found_in_text","path":"run_models/calculate_surprisal.py","file_url":"https://github.com/jmichaelov/lm-behavioral-phases/blob/HEAD/run_models/calculate_surprisal.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4ab3a7754330400c"}},{"code_sha256_prefix":"6139913fe16abc3a","entry":"is_valid_sentence","repo":"jmichaelov/lm-behavioral-phases","repo_kind":"found_in_text","path":"dataset_construction/construct_dataset1.py","file_url":"https://github.com/jmichaelov/lm-behavioral-phases/blob/HEAD/dataset_construction/construct_dataset1.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6139913fe16abc3a"}},{"code_sha256_prefix":"9d17716ba5776b95","entry":"mark","repo":"jmichaelov/lm-behavioral-phases","repo_kind":"found_in_text","path":"dataset_construction/construct_dataset3.py","file_url":"https://github.com/jmichaelov/lm-behavioral-phases/blob/HEAD/dataset_construction/construct_dataset3.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9d17716ba5776b95"}},{"code_sha256_prefix":"2f330a88ee04e562","entry":"process_args","repo":"jmichaelov/lm-behavioral-phases","repo_kind":"found_in_text","path":"run_models/calculate_surprisal.py","file_url":"https://github.com/jmichaelov/lm-behavioral-phases/blob/HEAD/run_models/calculate_surprisal.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2f330a88ee04e562"}},{"code_sha256_prefix":"53e8e82c6e6898a8","entry":"select_word","repo":"jmichaelov/lm-behavioral-phases","repo_kind":"found_in_text","path":"dataset_construction/construct_dataset2.py","file_url":"https://github.com/jmichaelov/lm-behavioral-phases/blob/HEAD/dataset_construction/construct_dataset2.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"53e8e82c6e6898a8"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.CL","source":"arxiv_api"},"syntology_extracted_results":null}