{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/breaking-the-activation-function-bottleneck","title":"Breaking the Activation Function Bottleneck through Adaptive Parameterization","arxiv_id":"1805.08574","date":"2018-05-22","proceeding":"NeurIPS 2018 12","authors":["Sebastian Flennerhag","Hujun Yin","John Keane","Mark Elliot"],"abstract":"Standard neural network architectures are non-linear only by virtue of a\nsimple element-wise activation function, making them both brittle and\nexcessively large. In this paper, we consider methods for making the\nfeed-forward layer more flexible while preserving its basic structure. We\ndevelop simple drop-in replacements that learn to adapt their parameterization\nconditional on the input, thereby increasing statistical efficiency\nsignificantly. We present an adaptive LSTM that advances the state of the art\nfor the Penn Treebank and WikiText-2 word-modeling tasks while using fewer\nparameters and converging in less than half as many iterations.","url_abs":"http://arxiv.org/abs/1805.08574v4","url_pdf":"http://arxiv.org/pdf/1805.08574v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"breaking-the-activation-function-bottleneck","repo_url":"https://github.com/flennerhag/alstm","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"BSD-3-Clause"}}],"tasks":[],"methods":[{"method_slug":"lstm","method_name":"LSTM"},{"method_slug":"sigmoid-activation","method_name":"Sigmoid Activation"},{"method_slug":"tanh-activation","method_name":"Tanh Activation"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1805.08574","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1805.08574"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/flennerhag/alstm","reach":{"status":"ok","spdx":"BSD-3-Clause"}}],"summary":{"unverified":4},"by_repo_kind":{"official":{"samples":4,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"44f6fbff140a0097","entry":"arnn_cell","repo":"flennerhag/alstm","repo_kind":"official","path":"alstm/arnn.py","file_url":"https://github.com/flennerhag/alstm/blob/HEAD/alstm/arnn.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"44f6fbff140a0097"}},{"code_sha256_prefix":"9b1460555de721f1","entry":"chunk","repo":"flennerhag/alstm","repo_kind":"official","path":"alstm/utils.py","file_url":"https://github.com/flennerhag/alstm/blob/HEAD/alstm/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"9b1460555de721f1"}},{"code_sha256_prefix":"38a8f6346cf475e6","entry":"get_sizes","repo":"flennerhag/alstm","repo_kind":"official","path":"alstm/utils.py","file_url":"https://github.com/flennerhag/alstm/blob/HEAD/alstm/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"38a8f6346cf475e6"}},{"code_sha256_prefix":"474f0bb2de1a2950","entry":"init_hidden","repo":"flennerhag/alstm","repo_kind":"official","path":"alstm/utils.py","file_url":"https://github.com/flennerhag/alstm/blob/HEAD/alstm/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"474f0bb2de1a2950"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}