{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/breaking-the-softmax-bottleneck-a-high-rank","title":"Breaking the Softmax Bottleneck: A High-Rank RNN Language Model","arxiv_id":"1711.03953","date":"2017-11-10","proceeding":"ICLR 2018 1","authors":["Zhilin Yang","Zihang Dai","Ruslan Salakhutdinov","William W. Cohen"],"abstract":"We formulate language modeling as a matrix factorization problem, and show\nthat the expressiveness of Softmax-based models (including the majority of\nneural language models) is limited by a Softmax bottleneck. Given that natural\nlanguage is highly context-dependent, this further implies that in practice\nSoftmax with distributed word embeddings does not have enough capacity to model\nnatural language. We propose a simple and effective method to address this\nissue, and improve the state-of-the-art perplexities on Penn Treebank and\nWikiText-2 to 47.69 and 40.68 respectively. The proposed method also excels on\nthe large-scale 1B Word dataset, outperforming the baseline by over 5.6 points\nin perplexity.","url_abs":"http://arxiv.org/abs/1711.03953v4","url_pdf":"http://arxiv.org/pdf/1711.03953v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"breaking-the-softmax-bottleneck-a-high-rank","repo_url":"https://github.com/zihangdai/mos","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"breaking-the-softmax-bottleneck-a-high-rank","repo_url":"https://github.com/cstorm125/thai2fit","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"unanswered"}},{"paper_slug":"breaking-the-softmax-bottleneck-a-high-rank","repo_url":"https://github.com/nkcr/overlap-ml","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"BSD-3-Clause"}},{"paper_slug":"breaking-the-softmax-bottleneck-a-high-rank","repo_url":"https://github.com/nunezpaul/MNIST","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"breaking-the-softmax-bottleneck-a-high-rank","repo_url":"https://github.com/omerlux/NLP-PTB","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"breaking-the-softmax-bottleneck-a-high-rank","repo_url":"https://github.com/omerlux/Recurrent_Neural_Network_-_Part_2","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"breaking-the-softmax-bottleneck-a-high-rank","repo_url":"https://github.com/tdmeeste/SparseSeqModels","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"breaking-the-softmax-bottleneck-a-high-rank","repo_url":"https://github.com/yfreedomliTHU/mos-pytorch1.1","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"breaking-the-softmax-bottleneck-a-high-rank","repo_url":"https://github.com/zhangyaoyuan/GAN-Simplification","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":null}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"high","task_name":"Vocal Bursts Intensity Prediction"},{"task_slug":"word-embeddings","task_name":"Word Embeddings"}],"methods":[{"method_slug":"awd-lstm","method_name":"AWD-LSTM"},{"method_slug":"activation-regularization","method_name":"Activation Regularization"},{"method_slug":"dropconnect","method_name":"DropConnect"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"embedding-dropout","method_name":"Embedding Dropout"},{"method_slug":"lstm","method_name":"LSTM"},{"method_slug":"mixture-of-softmaxes","method_name":"Mixture of Softmaxes"},{"method_slug":"sigmoid-activation","method_name":"Sigmoid Activation"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"tanh-activation","method_name":"Tanh Activation"},{"method_slug":"temporal-activation-regularization","method_name":"Temporal Activation Regularization"},{"method_slug":"variational-dropout","method_name":"Variational Dropout"},{"method_slug":"weight-tying","method_name":"Weight Tying"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/language-modelling-on-penn-treebank-word","task":"Language Modelling","dataset":"Penn Treebank (Word Level)","model":"AWD-LSTM-MoS + dynamic eval","rank_in_archive_order":11,"of":43,"metrics":{"Params":"22M","Test perplexity":"47.69","Validation perplexity":"48.33"},"uses_additional_data":false},{"leaderboard":"/sota/language-modelling-on-penn-treebank-word","task":"Language Modelling","dataset":"Penn Treebank (Word Level)","model":"AWD-LSTM-MoS","rank_in_archive_order":20,"of":43,"metrics":{"Params":"22M","Test perplexity":"54.44","Validation perplexity":"56.54"},"uses_additional_data":false},{"leaderboard":"/sota/language-modelling-on-wikitext-2","task":"Language Modelling","dataset":"WikiText-2","model":"AWD-LSTM-MoS + dynamic eval","rank_in_archive_order":16,"of":38,"metrics":{"Number of params":"35M","Test perplexity":"40.68","Validation perplexity":"42.41"},"uses_additional_data":false},{"leaderboard":"/sota/language-modelling-on-wikitext-2","task":"Language Modelling","dataset":"WikiText-2","model":"AWD-LSTM-MoS","rank_in_archive_order":26,"of":38,"metrics":{"Number of params":"35M","Test perplexity":"61.45","Validation perplexity":"63.88"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1711.03953","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1711.03953"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yfreedomliTHU/mos-pytorch1.1","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/omerlux/NLP-PTB","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/cstorm125/thai2fit","reach":{"status":"unanswered"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/zhangyaoyuan/GAN-Simplification","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/nunezpaul/MNIST","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/nkcr/overlap-ml","reach":{"status":"ok","spdx":"BSD-3-Clause"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/omerlux/Recurrent_Neural_Network_-_Part_2","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/tdmeeste/SparseSeqModels","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/zihangdai/mos","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":1,"unverified":22},"by_repo_kind":{"official":{"samples":3,"ran":0,"repositories":1},"listed":{"samples":20,"ran":1,"repositories":4}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"2943dadcb85e6e3c","entry":"batchify","repo":"yfreedomliTHU/mos-pytorch1.1","repo_kind":"listed","path":"utils.py","file_url":"https://github.com/yfreedomliTHU/mos-pytorch1.1/blob/HEAD/utils.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2943dadcb85e6e3c"}},{"code_sha256_prefix":"62876a934ad2732b","entry":"build_vocab","repo":"omerlux/NLP-PTB","repo_kind":"listed","path":"Language_Modeling_Old/Language_Modeling_gen.py","file_url":"https://github.com/omerlux/NLP-PTB/blob/HEAD/Language_Modeling_Old/Language_Modeling_gen.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"62876a934ad2732b"}},{"code_sha256_prefix":"a2f146bb53bb255e","entry":"cal_bleu","repo":"omerlux/NLP-PTB","repo_kind":"listed","path":"mos-pytorch1.1/bleu.py","file_url":"https://github.com/omerlux/NLP-PTB/blob/HEAD/mos-pytorch1.1/bleu.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a2f146bb53bb255e"}},{"code_sha256_prefix":"c75648e7493bfa87","entry":"data_loader","repo":"omerlux/NLP-PTB","repo_kind":"listed","path":"Language_Modeling_DataRearange/Language_Modeling.py","file_url":"https://github.com/omerlux/NLP-PTB/blob/HEAD/Language_Modeling_DataRearange/Language_Modeling.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c75648e7493bfa87"}},{"code_sha256_prefix":"b47a3b26a9d39f5f","entry":"data_loader","repo":"omerlux/NLP-PTB","repo_kind":"listed","path":"Language_Modeling_Old/Language_Modeling.py","file_url":"https://github.com/omerlux/NLP-PTB/blob/HEAD/Language_Modeling_Old/Language_Modeling.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b47a3b26a9d39f5f"}},{"code_sha256_prefix":"dd918220a90f8779","entry":"data_loader","repo":"omerlux/NLP-PTB","repo_kind":"listed","path":"Language_Modeling_Old/Language_Modeling_tfds.py","file_url":"https://github.com/omerlux/NLP-PTB/blob/HEAD/Language_Modeling_Old/Language_Modeling_tfds.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dd918220a90f8779"}},{"code_sha256_prefix":"61247fd8db2741cf","entry":"embedded_dropout","repo":"yfreedomliTHU/mos-pytorch1.1","repo_kind":"listed","path":"embed_regularize.py","file_url":"https://github.com/yfreedomliTHU/mos-pytorch1.1/blob/HEAD/embed_regularize.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"61247fd8db2741cf"}},{"code_sha256_prefix":"612788fb8fc53b62","entry":"embedded_dropout","repo":"zihangdai/mos","repo_kind":"official","path":"embed_regularize.py","file_url":"https://github.com/zihangdai/mos/blob/HEAD/embed_regularize.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"612788fb8fc53b62"}},{"code_sha256_prefix":"6a3d8df0b754bb01","entry":"embedded_dropout","repo":"nkcr/overlap-ml","repo_kind":"listed","path":"awd/embed_regularize.py","file_url":"https://github.com/nkcr/overlap-ml/blob/HEAD/awd/embed_regularize.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"6a3d8df0b754bb01"}},{"code_sha256_prefix":"73781c047e5bde5a","entry":"features_labels","repo":"omerlux/NLP-PTB","repo_kind":"listed","path":"Language_Modeling_DataRearange/Language_Modeling.py","file_url":"https://github.com/omerlux/NLP-PTB/blob/HEAD/Language_Modeling_DataRearange/Language_Modeling.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"73781c047e5bde5a"}},{"code_sha256_prefix":"1d654edced8659d4","entry":"features_labels","repo":"omerlux/NLP-PTB","repo_kind":"listed","path":"Language_Modeling_Old/Language_Modeling.py","file_url":"https://github.com/omerlux/NLP-PTB/blob/HEAD/Language_Modeling_Old/Language_Modeling.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1d654edced8659d4"}},{"code_sha256_prefix":"d78e2f03397f3d7c","entry":"features_labels","repo":"omerlux/NLP-PTB","repo_kind":"listed","path":"Language_Modeling_Old/Language_Modeling_tfds.py","file_url":"https://github.com/omerlux/NLP-PTB/blob/HEAD/Language_Modeling_Old/Language_Modeling_tfds.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d78e2f03397f3d7c"}},{"code_sha256_prefix":"cbfd3e2359406dd6","entry":"file_to_word_ids","repo":"omerlux/NLP-PTB","repo_kind":"listed","path":"Language_Modeling_Old/Language_Modeling_gen.py","file_url":"https://github.com/omerlux/NLP-PTB/blob/HEAD/Language_Modeling_Old/Language_Modeling_gen.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cbfd3e2359406dd6"}},{"code_sha256_prefix":"8585e034505891c4","entry":"get_batch","repo":"zihangdai/mos","repo_kind":"official","path":"utils.py","file_url":"https://github.com/zihangdai/mos/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8585e034505891c4"}},{"code_sha256_prefix":"591c3d72452ed508","entry":"get_batch","repo":"yfreedomliTHU/mos-pytorch1.1","repo_kind":"listed","path":"utils.py","file_url":"https://github.com/yfreedomliTHU/mos-pytorch1.1/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"591c3d72452ed508"}},{"code_sha256_prefix":"0180f4ce26db14fe","entry":"import_graph","repo":"nunezpaul/MNIST","repo_kind":"listed","path":"predict.py","file_url":"https://github.com/nunezpaul/MNIST/blob/HEAD/predict.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0180f4ce26db14fe"}},{"code_sha256_prefix":"c98fe0c21eb27db8","entry":"load_data","repo":"omerlux/NLP-PTB","repo_kind":"listed","path":"mos-pytorch1.1/bleu.py","file_url":"https://github.com/omerlux/NLP-PTB/blob/HEAD/mos-pytorch1.1/bleu.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c98fe0c21eb27db8"}},{"code_sha256_prefix":"7294b626ee6ccf20","entry":"model_create","repo":"omerlux/NLP-PTB","repo_kind":"listed","path":"Language_Modeling_DataRearange/Language_Modeling.py","file_url":"https://github.com/omerlux/NLP-PTB/blob/HEAD/Language_Modeling_DataRearange/Language_Modeling.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7294b626ee6ccf20"}},{"code_sha256_prefix":"1720dba9c0088a7c","entry":"model_create","repo":"omerlux/NLP-PTB","repo_kind":"listed","path":"Language_Modeling_Old/Language_Modeling.py","file_url":"https://github.com/omerlux/NLP-PTB/blob/HEAD/Language_Modeling_Old/Language_Modeling.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1720dba9c0088a7c"}},{"code_sha256_prefix":"6006a3916540a413","entry":"model_create","repo":"omerlux/NLP-PTB","repo_kind":"listed","path":"Language_Modeling_Old/Language_Modeling_tfds.py","file_url":"https://github.com/omerlux/NLP-PTB/blob/HEAD/Language_Modeling_Old/Language_Modeling_tfds.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6006a3916540a413"}},{"code_sha256_prefix":"8cfc87c86a68ccf9","entry":"one_hot","repo":"nkcr/overlap-ml","repo_kind":"listed","path":"awd/pointer.py","file_url":"https://github.com/nkcr/overlap-ml/blob/HEAD/awd/pointer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"8cfc87c86a68ccf9"}},{"code_sha256_prefix":"f5c5c321e38f81cd","entry":"read_words","repo":"omerlux/NLP-PTB","repo_kind":"listed","path":"Language_Modeling_Old/Language_Modeling_gen.py","file_url":"https://github.com/omerlux/NLP-PTB/blob/HEAD/Language_Modeling_Old/Language_Modeling_gen.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f5c5c321e38f81cd"}},{"code_sha256_prefix":"8bd3cf7e6979e441","entry":"repackage_hidden","repo":"zihangdai/mos","repo_kind":"official","path":"dynamiceval.py","file_url":"https://github.com/zihangdai/mos/blob/HEAD/dynamiceval.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8bd3cf7e6979e441"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}