{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/adaptive-input-representations-for-neural","title":"Adaptive Input Representations for Neural Language Modeling","arxiv_id":"1809.10853","date":"2018-09-28","proceeding":"ICLR 2019 5","authors":["Alexei Baevski","Michael Auli"],"abstract":"We introduce adaptive input representations for neural language modeling\nwhich extend the adaptive softmax of Grave et al. (2017) to input\nrepresentations of variable capacity. There are several choices on how to\nfactorize the input and output layers, and whether to model words, characters\nor sub-word units. We perform a systematic comparison of popular choices for a\nself-attentional architecture. Our experiments show that models equipped with\nadaptive embeddings are more than twice as fast to train than the popular\ncharacter input CNN while having a lower number of parameters. On the\nWikiText-103 benchmark we achieve 18.7 perplexity, an improvement of 10.5\nperplexity compared to the previously best published result and on the Billion\nWord benchmark, we achieve 23.02 perplexity.","url_abs":"http://arxiv.org/abs/1809.10853v3","url_pdf":"http://arxiv.org/pdf/1809.10853v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"adaptive-input-representations-for-neural","repo_url":"https://github.com/pytorch/fairseq","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":null},{"paper_slug":"adaptive-input-representations-for-neural","repo_url":"https://github.com/AranKomat/adapinp","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"adaptive-input-representations-for-neural","repo_url":"https://github.com/yuhao318/UP-ViT","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"}],"methods":[{"method_slug":"adaptive-input-representations","method_name":"Adaptive Input Representations"},{"method_slug":"adaptive-softmax","method_name":"Adaptive Softmax"},{"method_slug":"softmax","method_name":"Softmax"}],"datasets_introduced":[],"methods_introduced":[{"slug":"adaptive-input-representations","name":"Adaptive Input Representations","full_name":"Adaptive Input Representations"}],"results":[{"leaderboard":"/sota/language-modelling-on-one-billion-word","task":"Language Modelling","dataset":"One Billion Word","model":"Adaptive Input Very Large","rank_in_archive_order":7,"of":27,"metrics":{"Number of params":"1.0B","PPL":"23.02","Validation perplexity":"22.92"},"uses_additional_data":false},{"leaderboard":"/sota/language-modelling-on-one-billion-word","task":"Language Modelling","dataset":"One Billion Word","model":"Adaptive Input Large","rank_in_archive_order":11,"of":27,"metrics":{"Number of params":"0.46B","PPL":"23.91","Validation perplexity":"23.83"},"uses_additional_data":false},{"leaderboard":"/sota/language-modelling-on-wikitext-103","task":"Language Modelling","dataset":"WikiText-103","model":"Transformer (Adaptive inputs)","rank_in_archive_order":40,"of":89,"metrics":{"Number of params":"247M","Test perplexity":"18.70","Validation perplexity":"17.97"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1809.10853","mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}