{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/n-grammer-augmenting-transformers-with-latent-1","title":"N-Grammer: Augmenting Transformers with latent n-grams","arxiv_id":"2207.06366","date":"2022-07-13","proceeding":null,"authors":["Aurko Roy","Rohan Anil","Guangda Lai","Benjamin Lee","Jeffrey Zhao","Shuyuan Zhang","Shibo Wang","Ye Zhang","Shen Wu","Rigel Swavely","Tao","Yu","Phuong Dao","Christopher Fifty","Zhifeng Chen","Yonghui Wu"],"abstract":"Transformer models have recently emerged as one of the foundational models in natural language processing, and as a byproduct, there is significant recent interest and investment in scaling these models. However, the training and inference costs of these large Transformer language models are prohibitive, thus necessitating more research in identifying more efficient variants. In this work, we propose a simple yet effective modification to the Transformer architecture inspired by the literature in statistical language modeling, by augmenting the model with n-grams that are constructed from a discrete latent representation of the text sequence. We evaluate our model, the N-Grammer on language modeling on the C4 data-set as well as text classification on the SuperGLUE data-set, and find that it outperforms several strong baselines such as the Transformer and the Primer. We open-source our model for reproducibility purposes in Jax.","url_abs":"https://arxiv.org/abs/2207.06366v1","url_pdf":"https://arxiv.org/pdf/2207.06366v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"n-grammer-augmenting-transformers-with-latent-1","repo_url":"https://github.com/tensorflow/lingvo","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"tf","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"n-grammer-augmenting-transformers-with-latent-1","repo_url":"https://github.com/yiyixuxu/n-grammer-flax","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"common-sense-reasoning","task_name":"Common Sense Reasoning"},{"task_slug":"coreference-resolution","task_name":"Coreference Resolution"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"natural-language-inference","task_name":"Natural Language Inference"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"text-classification","task_name":"Text Classification"},{"task_slug":"word-sense-disambiguation","task_name":"Word Sense Disambiguation"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"depthwise-convolution","method_name":"Depthwise Convolution"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-dconv-head-attention","method_name":"Multi-DConv-Head Attention"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"primer","method_name":"Primer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"squared-relu","method_name":"Squared ReLU"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/common-sense-reasoning-on-record","task":"Common Sense Reasoning","dataset":"ReCoRD","model":"N-Grammer 343M","rank_in_archive_order":35,"of":45,"metrics":{"EM":"28.9","F1":"29.9"},"uses_additional_data":false},{"leaderboard":"/sota/coreference-resolution-on-winograd-schema","task":"Coreference Resolution","dataset":"Winograd Schema Challenge","model":"N-Grammer 343M","rank_in_archive_order":37,"of":82,"metrics":{"Accuracy":"68.3"},"uses_additional_data":false},{"leaderboard":"/sota/language-modelling-on-c4","task":"Language Modelling","dataset":"C4","model":"N-Grammer 343M","rank_in_archive_order":7,"of":9,"metrics":{"Perplexity":"14.79"},"uses_additional_data":false},{"leaderboard":"/sota/language-modelling-on-c4","task":"Language Modelling","dataset":"C4","model":"N-Grammer 288M","rank_in_archive_order":8,"of":9,"metrics":{"Perplexity":"15.01"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-commitmentbank","task":"Natural Language Inference","dataset":"CommitmentBank","model":"N-Grammer 343M","rank_in_archive_order":14,"of":20,"metrics":{"Accuracy":"67.9 ","F1":"59.7"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-rte","task":"Natural Language Inference","dataset":"RTE","model":"N-Grammer 343M","rank_in_archive_order":73,"of":90,"metrics":{"Accuracy":"59.2%"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-boolq","task":"Question Answering","dataset":"BoolQ","model":"N-Grammer 343M","rank_in_archive_order":47,"of":65,"metrics":{"Accuracy":"65"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-copa","task":"Question Answering","dataset":"COPA","model":"N-Grammer 343M","rank_in_archive_order":56,"of":60,"metrics":{"Accuracy":"60.0"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-multirc","task":"Question Answering","dataset":"MultiRC","model":"N-Grammer 343M","rank_in_archive_order":19,"of":30,"metrics":{"EM":"11.3","F1":"62"},"uses_additional_data":false},{"leaderboard":"/sota/word-sense-disambiguation-on-words-in-context","task":"Word Sense Disambiguation","dataset":"Words in Context","model":"N-Grammer 343M","rank_in_archive_order":22,"of":37,"metrics":{"Accuracy":"56.1"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2207.06366","atlas_url":"https://app.syntology.ai/?focus=2207.06366","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2207.06366"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yiyixuxu/n-grammer-flax","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/tensorflow/lingvo","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":6},"by_repo_kind":{"official":{"samples":5,"ran":5,"repositories":1},"listed":{"samples":1,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"6af7b11f92a34c6c","entry":"GetFlops","repo":"tensorflow/lingvo","repo_kind":"official","path":"lingvo/core/activations.py","file_url":"https://github.com/tensorflow/lingvo/blob/HEAD/lingvo/core/activations.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6af7b11f92a34c6c"}},{"code_sha256_prefix":"9ec960083b074b30","entry":"ImportAllParams","repo":"tensorflow/lingvo","repo_kind":"official","path":"lingvo/model_imports.py","file_url":"https://github.com/tensorflow/lingvo/blob/HEAD/lingvo/model_imports.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9ec960083b074b30"}},{"code_sha256_prefix":"ca4beda520280fb9","entry":"ImportParams","repo":"tensorflow/lingvo","repo_kind":"official","path":"lingvo/model_imports.py","file_url":"https://github.com/tensorflow/lingvo/blob/HEAD/lingvo/model_imports.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ca4beda520280fb9"}},{"code_sha256_prefix":"47af72600eb7ee27","entry":"get_bigram_ids","repo":"yiyixuxu/n-grammer-flax","repo_kind":"listed","path":"n_grammer_flax/n_grammer_flax.py","file_url":"https://github.com/yiyixuxu/n-grammer-flax/blob/HEAD/n_grammer_flax/n_grammer_flax.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"47af72600eb7ee27"}},{"code_sha256_prefix":"e1e9fb4c30469b3c","entry":"stateless_cache_dataset","repo":"tensorflow/lingvo","repo_kind":"official","path":"lingvo/compat.py","file_url":"https://github.com/tensorflow/lingvo/blob/HEAD/lingvo/compat.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e1e9fb4c30469b3c"}},{"code_sha256_prefix":"f1ff841d96a18fdc","entry":"stateless_shuffle_dataset","repo":"tensorflow/lingvo","repo_kind":"official","path":"lingvo/compat.py","file_url":"https://github.com/tensorflow/lingvo/blob/HEAD/lingvo/compat.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f1ff841d96a18fdc"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}