{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/a-law-of-next-token-prediction-in-large","title":"A Law of Next-Token Prediction in Large Language Models","arxiv_id":"2408.13442","date":"2024-08-24","proceeding":null,"authors":["Hangfeng He","Weijie J. Su"],"abstract":"Large language models (LLMs) have been widely employed across various application domains, yet their black-box nature poses significant challenges to understanding how these models process input data internally to make predictions. In this paper, we introduce a precise and quantitative law that governs the learning of contextualized token embeddings through intermediate layers in pre-trained LLMs for next-token prediction. Our findings reveal that each layer contributes equally to enhancing prediction accuracy, from the lowest to the highest layer -- a universal phenomenon observed across a diverse array of open-source LLMs, built on architectures such as Transformer, RWKV, and Mamba. We demonstrate that this law offers new perspectives and insights to inform and guide practices in LLM development and applications, including model scaling, pre-training tasks, and information flow. Overall, our law enables more fine-grained approaches to the design, training, and interpretation of LLMs through scrutinizing their internal data processing mechanisms.","url_abs":"https://arxiv.org/abs/2408.13442v1","url_pdf":"https://arxiv.org/pdf/2408.13442v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"a-law-of-next-token-prediction-in-large","repo_url":"https://github.com/hornhehhf/llm-ell","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"mamba","task_name":"Mamba"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"mamba","method_name":"Mamba"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2408.13442","atlas_url":"https://app.syntology.ai/?focus=2408.13442","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2408.13442"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/hornhehhf/llm-ell","reach":{"status":"ok"}}],"summary":{"ran":9,"unverified":1},"by_repo_kind":{"official":{"samples":10,"ran":9,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":10,"samples":[{"code_sha256_prefix":"731e74aa7479482a","entry":"get_class_features","repo":"hornhehhf/llm-ell","repo_kind":"official","path":"feature_quality_assessment.py","file_url":"https://github.com/hornhehhf/llm-ell/blob/HEAD/feature_quality_assessment.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"731e74aa7479482a"}},{"code_sha256_prefix":"1e8e910a04a79e5c","entry":"get_label_distribution","repo":"hornhehhf/llm-ell","repo_kind":"official","path":"feature_quality_assessment.py","file_url":"https://github.com/hornhehhf/llm-ell/blob/HEAD/feature_quality_assessment.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1e8e910a04a79e5c"}},{"code_sha256_prefix":"83945a82104027ee","entry":"get_separation_fuzziness","repo":"hornhehhf/llm-ell","repo_kind":"official","path":"feature_quality_assessment.py","file_url":"https://github.com/hornhehhf/llm-ell/blob/HEAD/feature_quality_assessment.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"83945a82104027ee"}},{"code_sha256_prefix":"6b7a283014ad42b4","entry":"linear_regression","repo":"hornhehhf/llm-ell","repo_kind":"official","path":"utils.py","file_url":"https://github.com/hornhehhf/llm-ell/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6b7a283014ad42b4"}},{"code_sha256_prefix":"2569a39582b28750","entry":"load_data_from_pickle","repo":"hornhehhf/llm-ell","repo_kind":"official","path":"utils.py","file_url":"https://github.com/hornhehhf/llm-ell/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2569a39582b28750"}},{"code_sha256_prefix":"8fe7cbd51bab2b6f","entry":"load_model","repo":"hornhehhf/llm-ell","repo_kind":"official","path":"feature_learning.py","file_url":"https://github.com/hornhehhf/llm-ell/blob/HEAD/feature_learning.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8fe7cbd51bab2b6f"}},{"code_sha256_prefix":"9c79d64508e5eb46","entry":"load_model_from_path","repo":"hornhehhf/llm-ell","repo_kind":"official","path":"feature_learning.py","file_url":"https://github.com/hornhehhf/llm-ell/blob/HEAD/feature_learning.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9c79d64508e5eb46"}},{"code_sha256_prefix":"9c5852b81b07fb2f","entry":"round_to_first_nonzero","repo":"hornhehhf/llm-ell","repo_kind":"official","path":"utils.py","file_url":"https://github.com/hornhehhf/llm-ell/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9c5852b81b07fb2f"}},{"code_sha256_prefix":"d69bd99d23c2f471","entry":"select_tokens","repo":"hornhehhf/llm-ell","repo_kind":"official","path":"visualization.py","file_url":"https://github.com/hornhehhf/llm-ell/blob/HEAD/visualization.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d69bd99d23c2f471"}},{"code_sha256_prefix":"5c46ee109865c7ed","entry":"sample_sentences","repo":"hornhehhf/llm-ell","repo_kind":"official","path":"feature_learning.py","file_url":"https://github.com/hornhehhf/llm-ell/blob/HEAD/feature_learning.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5c46ee109865c7ed"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}