{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/improving-language-models-by-retrieving-from","title":"Improving language models by retrieving from trillions of tokens","arxiv_id":"2112.04426","date":"2021-12-08","proceeding":null,"authors":["Sebastian Borgeaud","Arthur Mensch","Jordan Hoffmann","Trevor Cai","Eliza Rutherford","Katie Millican","George van den Driessche","Jean-Baptiste Lespiau","Bogdan Damoc","Aidan Clark","Diego de Las Casas","Aurelia Guy","Jacob Menick","Roman Ring","Tom Hennigan","Saffron Huang","Loren Maggiore","Chris Jones","Albin Cassirer","Andy Brock","Michela Paganini","Geoffrey Irving","Oriol Vinyals","Simon Osindero","Karen Simonyan","Jack W. Rae","Erich Elsen","Laurent SIfre"],"abstract":"We enhance auto-regressive language models by conditioning on document chunks retrieved from a large corpus, based on local similarity with preceding tokens. With a $2$ trillion token database, our Retrieval-Enhanced Transformer (RETRO) obtains comparable performance to GPT-3 and Jurassic-1 on the Pile, despite using 25$\\times$ fewer parameters. After fine-tuning, RETRO performance translates to downstream knowledge-intensive tasks such as question answering. RETRO combines a frozen Bert retriever, a differentiable encoder and a chunked cross-attention mechanism to predict tokens based on an order of magnitude more data than what is typically consumed during training. We typically train RETRO from scratch, yet can also rapidly RETROfit pre-trained transformers with retrieval and still achieve good performance. Our work opens up new avenues for improving language models through explicit memory at unprecedented scale.","url_abs":"https://arxiv.org/abs/2112.04426v3","url_pdf":"https://arxiv.org/pdf/2112.04426v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"improving-language-models-by-retrieving-from","repo_url":"https://github.com/lucidrains/RETRO-pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"improving-language-models-by-retrieving-from","repo_url":"https://github.com/labmlai/annotated_deep_learning_paper_implementations","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"retrieval","task_name":"Retrieval"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"cosine-annealing","method_name":"Cosine Annealing"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt-3","method_name":"GPT-3"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-cosine-annealing","method_name":"Linear Warmup With Cosine Annealing"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/language-modelling-on-wikitext-103","task":"Language Modelling","dataset":"WikiText-103","model":"RETRO (7.5B)","rank_in_archive_order":1,"of":89,"metrics":{"Number of params":"7532M","Test perplexity":"2.4"},"uses_additional_data":true},{"leaderboard":"/sota/question-answering-on-natural-questions","task":"Question Answering","dataset":"Natural Questions","model":"RETRO + DPR (full)","rank_in_archive_order":15,"of":47,"metrics":{"EM":"45.5"},"uses_additional_data":false}],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2112.04426","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2112.04426"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/lucidrains/RETRO-pytorch","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/labmlai/annotated_deep_learning_paper_implementations","reach":null}],"summary":{"ran":11,"ran_draft_wrong":2,"ran_violates":3,"unverified":7},"by_repo_kind":{"listed":{"samples":20,"ran":13,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":3,"samples":[{"code_sha256_prefix":"03b81e2f9a4d2d4c","entry":"Attention","repo":"lucidrains/RETRO-pytorch","repo_kind":"listed","path":"retro_pytorch/retro_pytorch.py","file_url":"https://github.com/lucidrains/RETRO-pytorch/blob/HEAD/retro_pytorch/retro_pytorch.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"03b81e2f9a4d2d4c"}},{"code_sha256_prefix":"f393e5da1f357aea","entry":"ChunkedCrossAttention","repo":"labmlai/annotated_deep_learning_paper_implementations","repo_kind":"listed","path":"labml_nn/transformers/retro/model.py","file_url":"https://github.com/labmlai/annotated_deep_learning_paper_implementations/blob/HEAD/labml_nn/transformers/retro/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f393e5da1f357aea"}},{"code_sha256_prefix":"c3e56d6f2af858c1","entry":"CrossAttention","repo":"labmlai/annotated_deep_learning_paper_implementations","repo_kind":"listed","path":"labml_nn/transformers/retro/model.py","file_url":"https://github.com/labmlai/annotated_deep_learning_paper_implementations/blob/HEAD/labml_nn/transformers/retro/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c3e56d6f2af858c1"}},{"code_sha256_prefix":"38be50d72d5874d6","entry":"FeedForward","repo":"lucidrains/RETRO-pytorch","repo_kind":"listed","path":"retro_pytorch/retro_pytorch.py","file_url":"https://github.com/lucidrains/RETRO-pytorch/blob/HEAD/retro_pytorch/retro_pytorch.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"38be50d72d5874d6"}},{"code_sha256_prefix":"1fae1fc6f059aacd","entry":"FeedForward","repo":"labmlai/annotated_deep_learning_paper_implementations","repo_kind":"listed","path":"labml_nn/transformers/retro/model.py","file_url":"https://github.com/labmlai/annotated_deep_learning_paper_implementations/blob/HEAD/labml_nn/transformers/retro/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1fae1fc6f059aacd"}},{"code_sha256_prefix":"562d2542a1eaacef","entry":"PostNorm","repo":"lucidrains/RETRO-pytorch","repo_kind":"listed","path":"retro_pytorch/retro_pytorch.py","file_url":"https://github.com/lucidrains/RETRO-pytorch/blob/HEAD/retro_pytorch/retro_pytorch.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"562d2542a1eaacef"}},{"code_sha256_prefix":"f44be7c955b39fa2","entry":"PreNorm","repo":"lucidrains/RETRO-pytorch","repo_kind":"listed","path":"retro_pytorch/retro_pytorch.py","file_url":"https://github.com/lucidrains/RETRO-pytorch/blob/HEAD/retro_pytorch/retro_pytorch.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f44be7c955b39fa2"}},{"code_sha256_prefix":"a4ae5f96225b96f3","entry":"RMSNorm","repo":"lucidrains/RETRO-pytorch","repo_kind":"listed","path":"retro_pytorch/retro_pytorch.py","file_url":"https://github.com/lucidrains/RETRO-pytorch/blob/HEAD/retro_pytorch/retro_pytorch.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a4ae5f96225b96f3"}},{"code_sha256_prefix":"9fbc41d552de00c6","entry":"RotaryEmbedding","repo":"lucidrains/RETRO-pytorch","repo_kind":"listed","path":"retro_pytorch/retro_pytorch.py","file_url":"https://github.com/lucidrains/RETRO-pytorch/blob/HEAD/retro_pytorch/retro_pytorch.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9fbc41d552de00c6"}},{"code_sha256_prefix":"6d63acdd4f116570","entry":"RotaryPositionalEmbeddings","repo":"labmlai/annotated_deep_learning_paper_implementations","repo_kind":"listed","path":"labml_nn/transformers/retro/model.py","file_url":"https://github.com/labmlai/annotated_deep_learning_paper_implementations/blob/HEAD/labml_nn/transformers/retro/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6d63acdd4f116570"}},{"code_sha256_prefix":"4c4fa4155f57fc48","entry":"SelfAttention","repo":"labmlai/annotated_deep_learning_paper_implementations","repo_kind":"listed","path":"labml_nn/transformers/retro/model.py","file_url":"https://github.com/labmlai/annotated_deep_learning_paper_implementations/blob/HEAD/labml_nn/transformers/retro/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4c4fa4155f57fc48"}},{"code_sha256_prefix":"a6120b0228dbf889","entry":"apply_rotary_pos_emb","repo":"lucidrains/RETRO-pytorch","repo_kind":"listed","path":"retro_pytorch/retro_pytorch.py","file_url":"https://github.com/lucidrains/RETRO-pytorch/blob/HEAD/retro_pytorch/retro_pytorch.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a6120b0228dbf889"}},{"code_sha256_prefix":"75e2de7f7799802d","entry":"cast_tuple","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"75e2de7f7799802d"}},{"code_sha256_prefix":"60fff7c3c400d7ff","entry":"default","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"60fff7c3c400d7ff"}},{"code_sha256_prefix":"aa5486a3650902d8","entry":"exists","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"aa5486a3650902d8"}},{"code_sha256_prefix":"fdd93453f92f9167","entry":"rotate_half","repo":"lucidrains/RETRO-pytorch","repo_kind":"listed","path":"retro_pytorch/retro_pytorch.py","file_url":"https://github.com/lucidrains/RETRO-pytorch/blob/HEAD/retro_pytorch/retro_pytorch.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fdd93453f92f9167"}},{"code_sha256_prefix":"a18eb1addcd13eca","entry":"ChunkedCrossAttention","repo":"lucidrains/RETRO-pytorch","repo_kind":"listed","path":"retro_pytorch/retro_pytorch.py","file_url":"https://github.com/lucidrains/RETRO-pytorch/blob/HEAD/retro_pytorch/retro_pytorch.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a18eb1addcd13eca"}},{"code_sha256_prefix":"df5ec3eb8c24bd01","entry":"Decoder","repo":"lucidrains/RETRO-pytorch","repo_kind":"listed","path":"retro_pytorch/retro_pytorch.py","file_url":"https://github.com/lucidrains/RETRO-pytorch/blob/HEAD/retro_pytorch/retro_pytorch.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"df5ec3eb8c24bd01"}},{"code_sha256_prefix":"dfd283b4f5317daf","entry":"Encoder","repo":"lucidrains/RETRO-pytorch","repo_kind":"listed","path":"retro_pytorch/retro_pytorch.py","file_url":"https://github.com/lucidrains/RETRO-pytorch/blob/HEAD/retro_pytorch/retro_pytorch.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"dfd283b4f5317daf"}},{"code_sha256_prefix":"ed827370476041d7","entry":"NearestNeighborEncoder","repo":"labmlai/annotated_deep_learning_paper_implementations","repo_kind":"listed","path":"labml_nn/transformers/retro/model.py","file_url":"https://github.com/labmlai/annotated_deep_learning_paper_implementations/blob/HEAD/labml_nn/transformers/retro/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ed827370476041d7"}},{"code_sha256_prefix":"c136963c983ee0fb","entry":"RETRO","repo":"lucidrains/RETRO-pytorch","repo_kind":"listed","path":"retro_pytorch/retro_pytorch.py","file_url":"https://github.com/lucidrains/RETRO-pytorch/blob/HEAD/retro_pytorch/retro_pytorch.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c136963c983ee0fb"}},{"code_sha256_prefix":"5156ed7e3c0f7fef","entry":"RetroModel","repo":"labmlai/annotated_deep_learning_paper_implementations","repo_kind":"listed","path":"labml_nn/transformers/retro/model.py","file_url":"https://github.com/labmlai/annotated_deep_learning_paper_implementations/blob/HEAD/labml_nn/transformers/retro/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5156ed7e3c0f7fef"}},{"code_sha256_prefix":"2f92b91f3350e0c9","entry":"deepnorm_init","repo":"lucidrains/RETRO-pytorch","repo_kind":"listed","path":"retro_pytorch/retro_pytorch.py","file_url":"https://github.com/lucidrains/RETRO-pytorch/blob/HEAD/retro_pytorch/retro_pytorch.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2f92b91f3350e0c9"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}