{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/online-normalization-for-training-neural","title":"Online Normalization for Training Neural Networks","arxiv_id":"1905.05894","date":"2019-05-15","proceeding":"NeurIPS 2019 12","authors":["Vitaliy Chiley","Ilya Sharapov","Atli Kosson","Urs Koster","Ryan Reece","Sofia Samaniego de la Fuente","Vishal Subbiah","Michael James"],"abstract":"Online Normalization is a new technique for normalizing the hidden activations of a neural network. Like Batch Normalization, it normalizes the sample dimension. While Online Normalization does not use batches, it is as accurate as Batch Normalization. We resolve a theoretical limitation of Batch Normalization by introducing an unbiased technique for computing the gradient of normalized activations. Online Normalization works with automatic differentiation by adding statistical normalization as a primitive. This technique can be used in cases not covered by some other normalizers, such as recurrent networks, fully connected networks, and networks with activation memory requirements prohibitive for batching. We show its applications to image classification, image segmentation, and language modeling. We present formal proofs and experimental results on ImageNet, CIFAR, and PTB datasets.","url_abs":"https://arxiv.org/abs/1905.05894v3","url_pdf":"https://arxiv.org/pdf/1905.05894v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"online-normalization-for-training-neural","repo_url":"https://github.com/cerebras/online-normalization","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"BSD-3-Clause"}}],"tasks":[{"task_slug":"classification","task_name":"General Classification"},{"task_slug":"image-classification","task_name":"Image Classification"},{"task_slug":"image-segmentation","task_name":"Image Segmentation"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"},{"task_slug":"image-classification","task_name":"image-classification"}],"methods":[{"method_slug":"1x1-convolution","method_name":"1x1 Convolution"},{"method_slug":"average-pooling","method_name":"Average Pooling"},{"method_slug":"batch-normalization","method_name":"Batch Normalization"},{"method_slug":"bottleneck-residual-block","method_name":"Bottleneck Residual Block"},{"method_slug":"convolution","method_name":"Convolution"},{"method_slug":"global-average-pooling","method_name":"Global Average Pooling"},{"method_slug":"group-normalization","method_name":"Group Normalization"},{"method_slug":"instance-normalization","method_name":"Instance Normalization"},{"method_slug":"kaiming-initialization","method_name":"Kaiming Initialization"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"max-pooling","method_name":"Max Pooling"},{"method_slug":"onlinenorm","method_name":"Online Normalization"},{"method_slug":"relu","method_name":"ReLU"},{"method_slug":"residual-block","method_name":"Residual Block"},{"method_slug":"residual-connection","method_name":"Residual Connection"}],"datasets_introduced":[],"methods_introduced":[{"slug":"onlinenorm","name":"Online Normalization","full_name":"Online Normalization"}],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1905.05894","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1905.05894"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/cerebras/online-normalization","reach":{"status":"ok","spdx":"BSD-3-Clause"}}],"summary":{"unverified":16},"by_repo_kind":{"official":{"samples":16,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"23d0bd9f1b241feb","entry":"accuracy","repo":"cerebras/online-normalization","repo_kind":"official","path":"experiments/resnet/utils.py","file_url":"https://github.com/cerebras/online-normalization/blob/HEAD/experiments/resnet/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"23d0bd9f1b241feb"}},{"code_sha256_prefix":"30240401a8068dfc","entry":"add_square","repo":"cerebras/online-normalization","repo_kind":"official","path":"experiments/unet/simulation.py","file_url":"https://github.com/cerebras/online-normalization/blob/HEAD/experiments/unet/simulation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"30240401a8068dfc"}},{"code_sha256_prefix":"50f69e4173cfbf3b","entry":"conv_alongb_w1","repo":"cerebras/online-normalization","repo_kind":"official","path":"online-norm/pytorch/online_norm_pytorch/online_norm_1d.py","file_url":"https://github.com/cerebras/online-normalization/blob/HEAD/online-norm/pytorch/online_norm_pytorch/online_norm_1d.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"50f69e4173cfbf3b"}},{"code_sha256_prefix":"cf4866a0e5413a8e","entry":"conv_alongb_w1","repo":"cerebras/online-normalization","repo_kind":"official","path":"online-norm/pytorch/online_norm_pytorch/online_norm_2d.py","file_url":"https://github.com/cerebras/online-normalization/blob/HEAD/online-norm/pytorch/online_norm_pytorch/online_norm_2d.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"cf4866a0e5413a8e"}},{"code_sha256_prefix":"2840eb73965f941b","entry":"generate_img_and_mask","repo":"cerebras/online-normalization","repo_kind":"official","path":"experiments/unet/simulation.py","file_url":"https://github.com/cerebras/online-normalization/blob/HEAD/experiments/unet/simulation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"2840eb73965f941b"}},{"code_sha256_prefix":"42517d21b4bbbe1d","entry":"generate_random_data","repo":"cerebras/online-normalization","repo_kind":"official","path":"experiments/unet/simulation.py","file_url":"https://github.com/cerebras/online-normalization/blob/HEAD/experiments/unet/simulation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"42517d21b4bbbe1d"}},{"code_sha256_prefix":"d8bd6382fa212764","entry":"get_parameter_groups","repo":"cerebras/online-normalization","repo_kind":"official","path":"experiments/resnet/utils.py","file_url":"https://github.com/cerebras/online-normalization/blob/HEAD/experiments/resnet/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"d8bd6382fa212764"}},{"code_sha256_prefix":"c3c01085619cd9fa","entry":"lin_crtl","repo":"cerebras/online-normalization","repo_kind":"official","path":"online-norm/pytorch/online_norm_pytorch/online_norm_1d.py","file_url":"https://github.com/cerebras/online-normalization/blob/HEAD/online-norm/pytorch/online_norm_pytorch/online_norm_1d.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"c3c01085619cd9fa"}},{"code_sha256_prefix":"1ab49f7983db64d0","entry":"lin_momentum","repo":"cerebras/online-normalization","repo_kind":"official","path":"online-norm/pytorch/online_norm_pytorch/online_norm_1d.py","file_url":"https://github.com/cerebras/online-normalization/blob/HEAD/online-norm/pytorch/online_norm_pytorch/online_norm_1d.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"1ab49f7983db64d0"}},{"code_sha256_prefix":"846d691e31cde436","entry":"lin_momentum","repo":"cerebras/online-normalization","repo_kind":"official","path":"online-norm/pytorch/online_norm_pytorch/online_norm_2d.py","file_url":"https://github.com/cerebras/online-normalization/blob/HEAD/online-norm/pytorch/online_norm_pytorch/online_norm_2d.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"846d691e31cde436"}},{"code_sha256_prefix":"4994246847dd8bb1","entry":"mean_tensor","repo":"cerebras/online-normalization","repo_kind":"official","path":"online-norm/pytorch/online_norm_pytorch/online_norm_2d.py","file_url":"https://github.com/cerebras/online-normalization/blob/HEAD/online-norm/pytorch/online_norm_pytorch/online_norm_2d.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"4994246847dd8bb1"}},{"code_sha256_prefix":"d29637ee5873d351","entry":"mult_scale_forward","repo":"cerebras/online-normalization","repo_kind":"official","path":"online-norm/numpy_on/online_norm_1d.py","file_url":"https://github.com/cerebras/online-normalization/blob/HEAD/online-norm/numpy_on/online_norm_1d.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"d29637ee5873d351"}},{"code_sha256_prefix":"bf2eb181871a83c7","entry":"norm_backward","repo":"cerebras/online-normalization","repo_kind":"official","path":"online-norm/numpy_on/online_norm_1d.py","file_url":"https://github.com/cerebras/online-normalization/blob/HEAD/online-norm/numpy_on/online_norm_1d.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"bf2eb181871a83c7"}},{"code_sha256_prefix":"820691136676a1ce","entry":"norm_forward","repo":"cerebras/online-normalization","repo_kind":"official","path":"online-norm/numpy_on/online_norm_1d.py","file_url":"https://github.com/cerebras/online-normalization/blob/HEAD/online-norm/numpy_on/online_norm_1d.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"820691136676a1ce"}},{"code_sha256_prefix":"60e3e0091d896b87","entry":"norm_forward","repo":"cerebras/online-normalization","repo_kind":"official","path":"online-norm/numpy_on/online_norm_2d.py","file_url":"https://github.com/cerebras/online-normalization/blob/HEAD/online-norm/numpy_on/online_norm_2d.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"60e3e0091d896b87"}},{"code_sha256_prefix":"80a8d95be697182f","entry":"validate","repo":"cerebras/online-normalization","repo_kind":"official","path":"experiments/resnet/utils.py","file_url":"https://github.com/cerebras/online-normalization/blob/HEAD/experiments/resnet/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"80a8d95be697182f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}