{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/learning-dynamics-of-linear-denoising","title":"Learning Dynamics of Linear Denoising Autoencoders","arxiv_id":"1806.05413","date":"2018-06-14","proceeding":"ICML 2018 7","authors":["Arnu Pretorius","Steve Kroon","Herman Kamper"],"abstract":"Denoising autoencoders (DAEs) have proven useful for unsupervised\nrepresentation learning, but a thorough theoretical understanding is still\nlacking of how the input noise influences learning. Here we develop theory for\nhow noise influences learning in DAEs. By focusing on linear DAEs, we are able\nto derive analytic expressions that exactly describe their learning dynamics.\nWe verify our theoretical predictions with simulations as well as experiments\non MNIST and CIFAR-10. The theory illustrates how, when tuned correctly, noise\nallows DAEs to ignore low variance directions in the inputs while learning to\nreconstruct them. Furthermore, in a comparison of the learning dynamics of DAEs\nto standard regularised autoencoders, we show that noise has a similar\nregularisation effect to weight decay, but with faster training dynamics. We\nalso show that our theoretical predictions approximate learning dynamics on\nreal-world data and qualitatively match observed dynamics in nonlinear DAEs.","url_abs":"http://arxiv.org/abs/1806.05413v2","url_pdf":"http://arxiv.org/pdf/1806.05413v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"learning-dynamics-of-linear-denoising","repo_url":"https://github.com/arnupretorius/lindaedynamics_icml2018","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"denoising","task_name":"Denoising"},{"task_slug":"representation-learning","task_name":"Representation Learning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/1806.05413","atlas_url":"https://app.syntology.ai/?focus=1806.05413","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1806.05413"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/arnupretorius/lindaedynamics_icml2018","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":19},"by_repo_kind":{"official":{"samples":19,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"14f8157279eda080","entry":"affine_backward","repo":"arnupretorius/lindaedynamics_icml2018","repo_kind":"official","path":"src/scalar_net/layers.py","file_url":"https://github.com/arnupretorius/lindaedynamics_icml2018/blob/HEAD/src/scalar_net/layers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"14f8157279eda080"}},{"code_sha256_prefix":"5f1182602605491b","entry":"affine_forward","repo":"arnupretorius/lindaedynamics_icml2018","repo_kind":"official","path":"src/scalar_net/layers.py","file_url":"https://github.com/arnupretorius/lindaedynamics_icml2018/blob/HEAD/src/scalar_net/layers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5f1182602605491b"}},{"code_sha256_prefix":"e019ef114b73880b","entry":"animate_optimiser_trajectory_2d","repo":"arnupretorius/lindaedynamics_icml2018","repo_kind":"official","path":"src/scalar_net/visualisations.py","file_url":"https://github.com/arnupretorius/lindaedynamics_icml2018/blob/HEAD/src/scalar_net/visualisations.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e019ef114b73880b"}},{"code_sha256_prefix":"94a23729745a0c4d","entry":"compute_correlation_matrix","repo":"arnupretorius/lindaedynamics_icml2018","repo_kind":"official","path":"src/linear_ae_net/dynamics.py","file_url":"https://github.com/arnupretorius/lindaedynamics_icml2018/blob/HEAD/src/linear_ae_net/dynamics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"94a23729745a0c4d"}},{"code_sha256_prefix":"09cd88e02a7b5179","entry":"create_model","repo":"arnupretorius/lindaedynamics_icml2018","repo_kind":"official","path":"src/nonlinear_ae_net/nonlinear_ae_net.py","file_url":"https://github.com/arnupretorius/lindaedynamics_icml2018/blob/HEAD/src/nonlinear_ae_net/nonlinear_ae_net.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"09cd88e02a7b5179"}},{"code_sha256_prefix":"9077a88c2c93afef","entry":"dae_learning_dynamics","repo":"arnupretorius/lindaedynamics_icml2018","repo_kind":"official","path":"src/linear_ae_net/dynamics.py","file_url":"https://github.com/arnupretorius/lindaedynamics_icml2018/blob/HEAD/src/linear_ae_net/dynamics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9077a88c2c93afef"}},{"code_sha256_prefix":"1936ca8c49dcffd2","entry":"generate_circle_points","repo":"arnupretorius/lindaedynamics_icml2018","repo_kind":"official","path":"src/scalar_net/utils.py","file_url":"https://github.com/arnupretorius/lindaedynamics_icml2018/blob/HEAD/src/scalar_net/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1936ca8c49dcffd2"}},{"code_sha256_prefix":"5676f4e445be4c1e","entry":"gradient_descent","repo":"arnupretorius/lindaedynamics_icml2018","repo_kind":"official","path":"src/scalar_net/optimisers.py","file_url":"https://github.com/arnupretorius/lindaedynamics_icml2018/blob/HEAD/src/scalar_net/optimisers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5676f4e445be4c1e"}},{"code_sha256_prefix":"456c9738bf586056","entry":"hyperbolic_learning_dynamics","repo":"arnupretorius/lindaedynamics_icml2018","repo_kind":"official","path":"src/scalar_net/utils.py","file_url":"https://github.com/arnupretorius/lindaedynamics_icml2018/blob/HEAD/src/scalar_net/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"456c9738bf586056"}},{"code_sha256_prefix":"5dd7f0a2a4002782","entry":"mean_squared_error_backward","repo":"arnupretorius/lindaedynamics_icml2018","repo_kind":"official","path":"src/scalar_net/loss_functions.py","file_url":"https://github.com/arnupretorius/lindaedynamics_icml2018/blob/HEAD/src/scalar_net/loss_functions.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5dd7f0a2a4002782"}},{"code_sha256_prefix":"495ac14b8cbc4cfc","entry":"mean_squared_error_forward","repo":"arnupretorius/lindaedynamics_icml2018","repo_kind":"official","path":"src/scalar_net/loss_functions.py","file_url":"https://github.com/arnupretorius/lindaedynamics_icml2018/blob/HEAD/src/scalar_net/loss_functions.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"495ac14b8cbc4cfc"}},{"code_sha256_prefix":"de719648984aa476","entry":"normal_init","repo":"arnupretorius/lindaedynamics_icml2018","repo_kind":"official","path":"src/scalar_net/initialisers.py","file_url":"https://github.com/arnupretorius/lindaedynamics_icml2018/blob/HEAD/src/scalar_net/initialisers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"de719648984aa476"}},{"code_sha256_prefix":"8af790e81422aab5","entry":"optimise","repo":"arnupretorius/lindaedynamics_icml2018","repo_kind":"official","path":"src/scalar_net/optimisers.py","file_url":"https://github.com/arnupretorius/lindaedynamics_icml2018/blob/HEAD/src/scalar_net/optimisers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8af790e81422aab5"}},{"code_sha256_prefix":"204b7e13e88d206c","entry":"plot_mse_loss_surface_2d","repo":"arnupretorius/lindaedynamics_icml2018","repo_kind":"official","path":"src/scalar_net/visualisations.py","file_url":"https://github.com/arnupretorius/lindaedynamics_icml2018/blob/HEAD/src/scalar_net/visualisations.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"204b7e13e88d206c"}},{"code_sha256_prefix":"4d3e418f768edb34","entry":"plot_mse_loss_surface_3d","repo":"arnupretorius/lindaedynamics_icml2018","repo_kind":"official","path":"src/scalar_net/visualisations.py","file_url":"https://github.com/arnupretorius/lindaedynamics_icml2018/blob/HEAD/src/scalar_net/visualisations.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4d3e418f768edb34"}},{"code_sha256_prefix":"4fe3aaa057cb62f9","entry":"relu_forward","repo":"arnupretorius/lindaedynamics_icml2018","repo_kind":"official","path":"src/scalar_net/layers.py","file_url":"https://github.com/arnupretorius/lindaedynamics_icml2018/blob/HEAD/src/scalar_net/layers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4fe3aaa057cb62f9"}},{"code_sha256_prefix":"a6b47b98df4d0ce7","entry":"theoretical_dynamics","repo":"arnupretorius/lindaedynamics_icml2018","repo_kind":"official","path":"src/scalar_net/utils.py","file_url":"https://github.com/arnupretorius/lindaedynamics_icml2018/blob/HEAD/src/scalar_net/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a6b47b98df4d0ce7"}},{"code_sha256_prefix":"de50c529f4b5188c","entry":"theoretical_learning_dynamics","repo":"arnupretorius/lindaedynamics_icml2018","repo_kind":"official","path":"src/linear_ae_net/dynamics.py","file_url":"https://github.com/arnupretorius/lindaedynamics_icml2018/blob/HEAD/src/linear_ae_net/dynamics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"de50c529f4b5188c"}},{"code_sha256_prefix":"3b15ad22ad03850b","entry":"uniform_init","repo":"arnupretorius/lindaedynamics_icml2018","repo_kind":"official","path":"src/scalar_net/initialisers.py","file_url":"https://github.com/arnupretorius/lindaedynamics_icml2018/blob/HEAD/src/scalar_net/initialisers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3b15ad22ad03850b"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}