{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2509-18344","title":"Speculate Deep and Accurate: Lossless and Training-Free Acceleration for Offloaded LLMs via Substitute Speculative Decoding","arxiv_id":"2509.18344","date":"2025-09-22","proceeding":"NeurIPS","authors":["Pei-Shuo Wang","Jian-Jia Chen","Chun-Che Yang","Chi-Chih Chang","Ning-Chi Huang","Mohamed S. Abdelfattah","Kai-Chiang Wu"],"abstract":"The immense model sizes of large language models (LLMs) challenge deployment on memory-limited consumer GPUs. Although model compression and parameter offloading are common strategies to address memory limitations, compression can degrade quality, and offloading maintains quality but suffers from slow inference. Speculative decoding presents a promising avenue to accelerate parameter offloading, utilizing a fast draft model to propose multiple draft tokens, which are then verified by the target LLM in parallel with a single forward pass. This method reduces the time-consuming data transfers in forward passes that involve offloaded weight transfers. Existing methods often rely on pretrained weights of the same family, but require additional training to align with custom-trained models. Moreover, approaches that involve draft model training usually yield only modest speedups. This limitation arises from insufficient alignment with the target model, preventing higher token acceptance lengths. To address these challenges and achieve greater speedups, we propose SubSpec, a plug-and-play method to accelerate parameter offloading that is lossless and training-free. SubSpec constructs a highly aligned draft model by generating low-bit quantized substitute layers from offloaded target LLM portions. Additionally, our method shares the remaining GPU-resident layers and the KV-Cache, further reducing memory overhead and enhance alignment. SubSpec achieves a high average acceptance length, delivering 9.1x speedup for Qwen2.5 7B on MT-Bench (8GB VRAM limit) and an average of 12.5x speedup for Qwen2.5 32B on popular generation benchmarks (24GB VRAM limit).","url_abs":"https://arxiv.org/abs/2509.18344","url_pdf":"https://arxiv.org/pdf/2509.18344","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2509.18344","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2509.18344"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/NYCU-EDgeAi/subspec","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"unverified":7},"by_repo_kind":{"found_in_text":{"samples":7,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"188b5813f0da549a","entry":"flatten_topk_outbuf","repo":"NYCU-EDgeAi/subspec","repo_kind":"found_in_text","path":"analysis/bench_topk_sampling.py","file_url":"https://github.com/NYCU-EDgeAi/subspec/blob/HEAD/analysis/bench_topk_sampling.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"188b5813f0da549a"}},{"code_sha256_prefix":"76ac9e07eb07f966","entry":"instantiate_recipe","repo":"NYCU-EDgeAi/subspec","repo_kind":"found_in_text","path":"run/core/config_utils.py","file_url":"https://github.com/NYCU-EDgeAi/subspec/blob/HEAD/run/core/config_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"76ac9e07eb07f966"}},{"code_sha256_prefix":"8a724c1da0cb0446","entry":"load_symbol","repo":"NYCU-EDgeAi/subspec","repo_kind":"found_in_text","path":"run/core/config_utils.py","file_url":"https://github.com/NYCU-EDgeAi/subspec/blob/HEAD/run/core/config_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8a724c1da0cb0446"}},{"code_sha256_prefix":"8989f7f258a815ab","entry":"old_flatten_topk","repo":"NYCU-EDgeAi/subspec","repo_kind":"found_in_text","path":"analysis/bench_topk_sampling.py","file_url":"https://github.com/NYCU-EDgeAi/subspec/blob/HEAD/analysis/bench_topk_sampling.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8989f7f258a815ab"}},{"code_sha256_prefix":"57be5bb247d6e444","entry":"share_param_deepcopy","repo":"NYCU-EDgeAi/subspec","repo_kind":"found_in_text","path":"specdecodes/models/draft_models/subspec_sd.py","file_url":"https://github.com/NYCU-EDgeAi/subspec/blob/HEAD/specdecodes/models/draft_models/subspec_sd.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"57be5bb247d6e444"}},{"code_sha256_prefix":"3910ab35ba400f25","entry":"two_stage_topk","repo":"NYCU-EDgeAi/subspec","repo_kind":"found_in_text","path":"analysis/bench_topk_sampling.py","file_url":"https://github.com/NYCU-EDgeAi/subspec/blob/HEAD/analysis/bench_topk_sampling.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3910ab35ba400f25"}},{"code_sha256_prefix":"0d393d7c772c88ce","entry":"write_settings_yaml","repo":"NYCU-EDgeAi/subspec","repo_kind":"found_in_text","path":"run/core/config_utils.py","file_url":"https://github.com/NYCU-EDgeAi/subspec/blob/HEAD/run/core/config_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0d393d7c772c88ce"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.CL","source":"arxiv_api"},"syntology_extracted_results":null}