{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/a-simple-way-to-initialize-recurrent-networks","title":"A Simple Way to Initialize Recurrent Networks of Rectified Linear Units","arxiv_id":"1504.00941","date":"2015-04-03","proceeding":null,"authors":["Quoc V. Le","Navdeep Jaitly","Geoffrey E. Hinton"],"abstract":"Learning long term dependencies in recurrent networks is difficult due to\nvanishing and exploding gradients. To overcome this difficulty, researchers\nhave developed sophisticated optimization techniques and network architectures.\nIn this paper, we propose a simpler solution that use recurrent neural networks\ncomposed of rectified linear units. Key to our solution is the use of the\nidentity matrix or its scaled version to initialize the recurrent weight\nmatrix. We find that our solution is comparable to LSTM on our four benchmarks:\ntwo toy problems involving long-range temporal structures, a large language\nmodeling problem and a benchmark speech recognition problem.","url_abs":"http://arxiv.org/abs/1504.00941v2","url_pdf":"http://arxiv.org/pdf/1504.00941v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"a-simple-way-to-initialize-recurrent-networks","repo_url":"https://github.com/adamx97/Data-Science-Advanced-Capstone","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"a-simple-way-to-initialize-recurrent-networks","repo_url":"https://github.com/facebookresearch/salina","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":null},{"paper_slug":"a-simple-way-to-initialize-recurrent-networks","repo_url":"https://github.com/mindspore-courses/DeepNLP-models-MindSpore","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"mindspore","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"a-simple-way-to-initialize-recurrent-networks","repo_url":"https://github.com/minhtriet/gridworld","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"a-simple-way-to-initialize-recurrent-networks","repo_url":"https://github.com/sobhan-moosavi/characterizingdrivingstyleswithdeeplearning","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"a-simple-way-to-initialize-recurrent-networks","repo_url":"https://github.com/trevor-richardson/rnn_zoo","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"sequential-image-classification","task_name":"Sequential Image Classification"},{"task_slug":"speech-recognition","task_name":"Speech Recognition"},{"task_slug":"speech-recognition-1","task_name":"speech-recognition"}],"methods":[{"method_slug":"lstm","method_name":"LSTM"},{"method_slug":"sigmoid-activation","method_name":"Sigmoid Activation"},{"method_slug":"tanh-activation","method_name":"Tanh Activation"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/sequential-image-classification-on-sequential","task":"Sequential Image Classification","dataset":"Sequential MNIST","model":"iRNN","rank_in_archive_order":27,"of":30,"metrics":{"Permuted Accuracy":"82%","Unpermuted Accuracy":"97%"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1504.00941","mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}