{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/ghost-noise-for-regularizing-deep-neural","title":"Ghost Noise for Regularizing Deep Neural Networks","arxiv_id":"2305.17205","date":"2023-05-26","proceeding":null,"authors":["Atli Kosson","Dongyang Fan","Martin Jaggi"],"abstract":"Batch Normalization (BN) is widely used to stabilize the optimization process and improve the test performance of deep neural networks. The regularization effect of BN depends on the batch size and explicitly using smaller batch sizes with Batch Normalization, a method known as Ghost Batch Normalization (GBN), has been found to improve generalization in many settings. We investigate the effectiveness of GBN by disentangling the induced ``Ghost Noise'' from normalization and quantitatively analyzing the distribution of noise as well as its impact on model performance. Inspired by our analysis, we propose a new regularization technique called Ghost Noise Injection (GNI) that imitates the noise in GBN without incurring the detrimental train-test discrepancy effects of small batch training. We experimentally show that GNI can provide a greater generalization benefit than GBN. Ghost Noise Injection can also be beneficial in otherwise non-noisy settings such as layer-normalized networks, providing additional evidence of the usefulness of Ghost Noise in Batch Normalization as a regularizer.","url_abs":"https://arxiv.org/abs/2305.17205v2","url_pdf":"https://arxiv.org/pdf/2305.17205v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"ghost-noise-for-regularizing-deep-neural","repo_url":"https://github.com/epfml/ghost-noise","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[],"methods":[{"method_slug":"batch-normalization","method_name":"Batch Normalization"},{"method_slug":"test","method_name":"Test"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2305.17205","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.17205"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/epfml/ghost-noise","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_honours":3,"ran":1,"unverified":2},"by_repo_kind":{"official":{"samples":6,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"410bffb8a07b52e3","entry":"count_params","repo":"epfml/ghost-noise","repo_kind":"official","path":"submodules/timm/benchmark.py","file_url":"https://github.com/epfml/ghost-noise/blob/HEAD/submodules/timm/benchmark.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":2,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"410bffb8a07b52e3"}},{"code_sha256_prefix":"33bca8cbc866bf64","entry":"checkpoint_metric","repo":"epfml/ghost-noise","repo_kind":"official","path":"submodules/timm/avg_checkpoints.py","file_url":"https://github.com/epfml/ghost-noise/blob/HEAD/submodules/timm/avg_checkpoints.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"33bca8cbc866bf64"}},{"code_sha256_prefix":"50d653ee6a59955c","entry":"cuda_timestamp","repo":"epfml/ghost-noise","repo_kind":"official","path":"submodules/timm/benchmark.py","file_url":"https://github.com/epfml/ghost-noise/blob/HEAD/submodules/timm/benchmark.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"50d653ee6a59955c"}},{"code_sha256_prefix":"475160c5a1a1225a","entry":"timestamp","repo":"epfml/ghost-noise","repo_kind":"official","path":"submodules/timm/benchmark.py","file_url":"https://github.com/epfml/ghost-noise/blob/HEAD/submodules/timm/benchmark.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"475160c5a1a1225a"}},{"code_sha256_prefix":"bec9456c146ecd1e","entry":"ghost_noise_injection","repo":"epfml/ghost-noise","repo_kind":"official","path":"nodo/minimal_gni.py","file_url":"https://github.com/epfml/ghost-noise/blob/HEAD/nodo/minimal_gni.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bec9456c146ecd1e"}},{"code_sha256_prefix":"3ec5d12d27b05a19","entry":"xbn2d","repo":"epfml/ghost-noise","repo_kind":"official","path":"nodo/exclusive_batch_norm.py","file_url":"https://github.com/epfml/ghost-noise/blob/HEAD/nodo/exclusive_batch_norm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3ec5d12d27b05a19"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}