{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/parameter-norm-growth-during-training-of","title":"Effects of Parameter Norm Growth During Transformer Training: Inductive Bias from Gradient Descent","arxiv_id":"2010.09697","date":"2020-10-19","proceeding":"EMNLP 2021 11","authors":["William Merrill","Vivek Ramanujan","Yoav Goldberg","Roy Schwartz","Noah Smith"],"abstract":"The capacity of neural networks like the widely adopted transformer is known to be very high. Evidence is emerging that they learn successfully due to inductive bias in the training routine, typically a variant of gradient descent (GD). To better understand this bias, we study the tendency for transformer parameters to grow in magnitude ($\\ell_2$ norm) during training, and its implications for the emergent representations within self attention layers. Empirically, we document norm growth in the training of transformer language models, including T5 during its pretraining. As the parameters grow in magnitude, we prove that the network approximates a discretized network with saturated activation functions. Such \"saturated\" networks are known to have a reduced capacity compared to the full network family that can be described in terms of formal languages and automata. Our results suggest saturation is a new characterization of an inductive bias implicit in GD of particular interest for NLP. We leverage the emergent discrete structure in a saturated transformer to analyze the role of different attention heads, finding that some focus locally on a small number of positions, while other heads compute global averages, allowing counting. We believe understanding the interplay between these two capabilities may shed further light on the structure of computation within large transformers.","url_abs":"https://arxiv.org/abs/2010.09697v5","url_pdf":"https://arxiv.org/pdf/2010.09697v5.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"parameter-norm-growth-during-training-of","repo_url":"https://github.com/viking-sudo-rm/norm-growth","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"inductive-bias","task_name":"Inductive Bias"}],"methods":[{"method_slug":"adafactor","method_name":"Adafactor"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"glu","method_name":"Gated Linear Unit"},{"method_slug":"inverse-square-root-schedule","method_name":"Inverse Square Root Schedule"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"relu","method_name":"ReLU"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"sentencepiece","method_name":"SentencePiece"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"t5","method_name":"T5"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2010.09697","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.09697"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/viking-sudo-rm/norm-growth","reach":null}],"summary":{"ran_draft_wrong":5,"ran_honours":1,"unverified":1},"by_repo_kind":{"official":{"samples":7,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":7,"samples":[{"code_sha256_prefix":"6198d16a5879a397","entry":"cos","repo":"viking-sudo-rm/norm-growth","repo_kind":"official","path":"eval_pretrain_sat_brown.py","file_url":"https://github.com/viking-sudo-rm/norm-growth/blob/HEAD/eval_pretrain_sat_brown.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6198d16a5879a397"}},{"code_sha256_prefix":"423851dbdaeb3755","entry":"filter_by_layer","repo":"viking-sudo-rm/norm-growth","repo_kind":"official","path":"t5_main.py","file_url":"https://github.com/viking-sudo-rm/norm-growth/blob/HEAD/t5_main.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"423851dbdaeb3755"}},{"code_sha256_prefix":"73b48f5f7899940d","entry":"get_checkpoints","repo":"viking-sudo-rm/norm-growth","repo_kind":"official","path":"t5_main.py","file_url":"https://github.com/viking-sudo-rm/norm-growth/blob/HEAD/t5_main.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"73b48f5f7899940d"}},{"code_sha256_prefix":"f255b17a12b2af42","entry":"get_norm","repo":"viking-sudo-rm/norm-growth","repo_kind":"official","path":"finetune_trans.py","file_url":"https://github.com/viking-sudo-rm/norm-growth/blob/HEAD/finetune_trans.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f255b17a12b2af42"}},{"code_sha256_prefix":"f3263e56c9d5df87","entry":"get_param_names","repo":"viking-sudo-rm/norm-growth","repo_kind":"official","path":"t5_main.py","file_url":"https://github.com/viking-sudo-rm/norm-growth/blob/HEAD/t5_main.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f3263e56c9d5df87"}},{"code_sha256_prefix":"ffc6543d0b4f8d4b","entry":"get_sentences","repo":"viking-sudo-rm/norm-growth","repo_kind":"official","path":"eval_pretrain_sat_brown.py","file_url":"https://github.com/viking-sudo-rm/norm-growth/blob/HEAD/eval_pretrain_sat_brown.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ffc6543d0b4f8d4b"}},{"code_sha256_prefix":"7d135ddd16507bbb","entry":"get_projection","repo":"viking-sudo-rm/norm-growth","repo_kind":"official","path":"finetune_trans.py","file_url":"https://github.com/viking-sudo-rm/norm-growth/blob/HEAD/finetune_trans.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7d135ddd16507bbb"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}