{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/beyond-autoregression-fast-llms-via-self","title":"Beyond Autoregression: Fast LLMs via Self-Distillation Through Time","arxiv_id":"2410.21035","date":"2024-10-28","proceeding":null,"authors":["Justin Deschenaux","Caglar Gulcehre"],"abstract":"Autoregressive (AR) Large Language Models (LLMs) have demonstrated significant success across numerous tasks. However, the AR modeling paradigm presents certain limitations; for instance, contemporary autoregressive LLMs are trained to generate one token at a time, which can result in noticeable latency. Recent advances have indicated that search and repeated sampling can enhance performance in various applications, such as theorem proving, code generation, and alignment, by utilizing greater computational resources during inference. In this study, we demonstrate that diffusion language models are capable of generating at least 32 tokens simultaneously, while exceeding the performance of AR models in text quality and on the LAMBADA natural language understanding benchmark. This outcome is achieved through a novel distillation method for discrete diffusion models, which reduces the number of inference steps by a factor of 32-64. Practically, our models, even without caching, can generate tokens at a rate that is up to 8 times faster than AR models employing KV caching, and we anticipate further improvements with the inclusion of caching. Moreover, we demonstrate the efficacy of our approach for diffusion language models with up to 860M parameters.","url_abs":"https://arxiv.org/abs/2410.21035v1","url_pdf":"https://arxiv.org/pdf/2410.21035v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"beyond-autoregression-fast-llms-via-self","repo_url":"https://github.com/jdeschena/sdtt","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"automated-theorem-proving","task_name":"Automated Theorem Proving"},{"task_slug":"code-generation","task_name":"Code Generation"},{"task_slug":"lambada","task_name":"LAMBADA"},{"task_slug":"natural-language-understanding","task_name":"Natural Language Understanding"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2410.21035","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.21035"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jdeschena/sdtt","reach":{"status":"ok"}}],"summary":{"ran_fixture":2,"ran":1,"ran_honours":1,"unverified":6},"by_repo_kind":{"official":{"samples":10,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":10,"samples":[{"code_sha256_prefix":"fbbc6722b74bda27","entry":"bias_dropout_add_scale","repo":"jdeschena/sdtt","repo_kind":"official","path":"src/sdtt/models/ar_orig.py","file_url":"https://github.com/jdeschena/sdtt/blob/HEAD/src/sdtt/models/ar_orig.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fbbc6722b74bda27"}},{"code_sha256_prefix":"ee894c881614c3ec","entry":"bias_dropout_add_scale_fused_train","repo":"jdeschena/sdtt","repo_kind":"official","path":"src/sdtt/models/ar_orig.py","file_url":"https://github.com/jdeschena/sdtt/blob/HEAD/src/sdtt/models/ar_orig.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ee894c881614c3ec"}},{"code_sha256_prefix":"06d2d8e50f91481f","entry":"get_bias_dropout_add_scale","repo":"jdeschena/sdtt","repo_kind":"official","path":"src/sdtt/models/ar_orig.py","file_url":"https://github.com/jdeschena/sdtt/blob/HEAD/src/sdtt/models/ar_orig.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"06d2d8e50f91481f"}},{"code_sha256_prefix":"78b555034273f3c7","entry":"modulate","repo":"jdeschena/sdtt","repo_kind":"official","path":"src/sdtt/models/dit_orig.py","file_url":"https://github.com/jdeschena/sdtt/blob/HEAD/src/sdtt/models/dit_orig.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"78b555034273f3c7"}},{"code_sha256_prefix":"b0a49b09dfb49452","entry":"eval_loop","repo":"jdeschena/sdtt","repo_kind":"official","path":"src/sdtt/harness_gpt2.py","file_url":"https://github.com/jdeschena/sdtt/blob/HEAD/src/sdtt/harness_gpt2.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b0a49b09dfb49452"}},{"code_sha256_prefix":"a7222ce47d825dbf","entry":"get_ckpt_idx","repo":"jdeschena/sdtt","repo_kind":"official","path":"src/sdtt/push_to_hub.py","file_url":"https://github.com/jdeschena/sdtt/blob/HEAD/src/sdtt/push_to_hub.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a7222ce47d825dbf"}},{"code_sha256_prefix":"0f2e39e89855eac7","entry":"gradient_norm","repo":"jdeschena/sdtt","repo_kind":"official","path":"src/sdtt/callbacks/gradient_norm.py","file_url":"https://github.com/jdeschena/sdtt/blob/HEAD/src/sdtt/callbacks/gradient_norm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0f2e39e89855eac7"}},{"code_sha256_prefix":"e23c2025aad80db2","entry":"parse_str_int_list","repo":"jdeschena/sdtt","repo_kind":"official","path":"src/sdtt/utils.py","file_url":"https://github.com/jdeschena/sdtt/blob/HEAD/src/sdtt/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e23c2025aad80db2"}},{"code_sha256_prefix":"8c79294eaaaef612","entry":"rm_null_values","repo":"jdeschena/sdtt","repo_kind":"official","path":"src/sdtt/utils.py","file_url":"https://github.com/jdeschena/sdtt/blob/HEAD/src/sdtt/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8c79294eaaaef612"}},{"code_sha256_prefix":"9811c7dc90993af3","entry":"str_to_dtype","repo":"jdeschena/sdtt","repo_kind":"official","path":"src/sdtt/utils.py","file_url":"https://github.com/jdeschena/sdtt/blob/HEAD/src/sdtt/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9811c7dc90993af3"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}