{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/rest-retrieval-based-speculative-decoding","title":"REST: Retrieval-Based Speculative Decoding","arxiv_id":"2311.08252","date":"2023-11-14","proceeding":null,"authors":["Zhenyu He","Zexuan Zhong","Tianle Cai","Jason D. Lee","Di He"],"abstract":"We introduce Retrieval-Based Speculative Decoding (REST), a novel algorithm designed to speed up language model generation. The key insight driving the development of REST is the observation that the process of text generation often includes certain common phases and patterns. Unlike previous methods that rely on a draft language model for speculative decoding, REST harnesses the power of retrieval to generate draft tokens. This method draws from the reservoir of existing knowledge, retrieving and employing relevant tokens based on the current context. Its plug-and-play nature allows for seamless integration and acceleration of any language models, all without necessitating additional training. When benchmarked on 7B and 13B language models in a single-batch setting, REST achieves a significant speedup of 1.62X to 2.36X on code or text generation. The code of REST is available at https://github.com/FasterDecoding/REST.","url_abs":"https://arxiv.org/abs/2311.08252v2","url_pdf":"https://arxiv.org/pdf/2311.08252v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"rest-retrieval-based-speculative-decoding","repo_url":"https://github.com/fasterdecoding/rest","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"text-generation","task_name":"Text Generation"}],"methods":[{"method_slug":"speed","method_name":"SPEED"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2311.08252","atlas_url":"https://app.syntology.ai/?focus=2311.08252","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.08252"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/fasterdecoding/rest","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":5,"ran_violates":1,"unverified":1},"by_repo_kind":{"official":{"samples":7,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"9fd8f39b4dc4957b","entry":"initialize_logits","repo":"fasterdecoding/rest","repo_kind":"official","path":"rest/model/utils.py","file_url":"https://github.com/fasterdecoding/rest/blob/HEAD/rest/model/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9fd8f39b4dc4957b"}},{"code_sha256_prefix":"3965aca677b30513","entry":"initialize_past_key_values","repo":"fasterdecoding/rest","repo_kind":"official","path":"rest/model/kv_cache.py","file_url":"https://github.com/fasterdecoding/rest/blob/HEAD/rest/model/kv_cache.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3965aca677b30513"}},{"code_sha256_prefix":"a0751eee3897a746","entry":"pad_path","repo":"fasterdecoding/rest","repo_kind":"official","path":"rest/model/utils.py","file_url":"https://github.com/fasterdecoding/rest/blob/HEAD/rest/model/utils.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a0751eee3897a746"}},{"code_sha256_prefix":"f6e0c0fc3868b787","entry":"repeat_kv","repo":"fasterdecoding/rest","repo_kind":"official","path":"rest/model/modeling_llama_kv.py","file_url":"https://github.com/fasterdecoding/rest/blob/HEAD/rest/model/modeling_llama_kv.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f6e0c0fc3868b787"}},{"code_sha256_prefix":"14f39e471f7c9e90","entry":"reset_past_key_values","repo":"fasterdecoding/rest","repo_kind":"official","path":"rest/model/utils.py","file_url":"https://github.com/fasterdecoding/rest/blob/HEAD/rest/model/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"14f39e471f7c9e90"}},{"code_sha256_prefix":"b71b0ed70aa2939c","entry":"rotate_half","repo":"fasterdecoding/rest","repo_kind":"official","path":"rest/model/modeling_llama_kv.py","file_url":"https://github.com/fasterdecoding/rest/blob/HEAD/rest/model/modeling_llama_kv.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b71b0ed70aa2939c"}},{"code_sha256_prefix":"bad260f6d71db00c","entry":"apply_rotary_pos_emb","repo":"fasterdecoding/rest","repo_kind":"official","path":"rest/model/modeling_llama_kv.py","file_url":"https://github.com/fasterdecoding/rest/blob/HEAD/rest/model/modeling_llama_kv.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bad260f6d71db00c"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}