{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/using-the-output-embedding-to-improve","title":"Using the Output Embedding to Improve Language Models","arxiv_id":"1608.05859","date":"2016-08-20","proceeding":"EACL 2017 4","authors":["Ofir Press","Lior Wolf"],"abstract":"We study the topmost weight matrix of neural network language models. We show\nthat this matrix constitutes a valid word embedding. When training language\nmodels, we recommend tying the input embedding and this output embedding. We\nanalyze the resulting update rules and show that the tied embedding evolves in\na more similar way to the output embedding than to the input embedding in the\nuntied model. We also offer a new method of regularizing the output embedding.\nOur methods lead to a significant reduction in perplexity, as we are able to\nshow on a variety of neural network language models. Finally, we show that\nweight tying can reduce the size of neural translation models to less than half\nof their original size without harming their performance.","url_abs":"http://arxiv.org/abs/1608.05859v3","url_pdf":"http://arxiv.org/pdf/1608.05859v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"using-the-output-embedding-to-improve","repo_url":"https://github.com/ofirpress/UsingTheOutputEmbedding","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"using-the-output-embedding-to-improve","repo_url":"https://github.com/BugOMan/summary_generator","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"using-the-output-embedding-to-improve","repo_url":"https://github.com/JeffreyJosanne/nematus_tf","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":null},{"paper_slug":"using-the-output-embedding-to-improve","repo_url":"https://github.com/arijitthegame/hybrid-sampling","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"using-the-output-embedding-to-improve","repo_url":"https://github.com/bhaddow/dev-nematus","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":null},{"paper_slug":"using-the-output-embedding-to-improve","repo_url":"https://github.com/faramarzmunshi/d2l-nlp","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"unanswered"}},{"paper_slug":"using-the-output-embedding-to-improve","repo_url":"https://github.com/floydhub/word-language-model","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"unanswered"}},{"paper_slug":"using-the-output-embedding-to-improve","repo_url":"https://github.com/jiali-ms/JLM","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":null},{"paper_slug":"using-the-output-embedding-to-improve","repo_url":"https://github.com/lium-lst/nmtpy","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"using-the-output-embedding-to-improve","repo_url":"https://github.com/zhangyaoyuan/GAN-Simplification","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":null}],"tasks":[{"task_slug":"translation","task_name":"Translation"},{"task_slug":null,"task_name":"valid"}],"methods":[{"method_slug":"weight-tying","method_name":"Weight Tying"}],"datasets_introduced":[],"methods_introduced":[{"slug":"weight-tying","name":"Weight Tying","full_name":"Weight Tying"}],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1608.05859","mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}