{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/infinigen-efficient-generative-inference-of","title":"InfiniGen: Efficient Generative Inference of Large Language Models with Dynamic KV Cache Management","arxiv_id":"2406.19707","date":"2024-06-28","proceeding":null,"authors":["Wonbeom Lee","Jungi Lee","Junghwan Seo","Jaewoong Sim"],"abstract":"Transformer-based large language models (LLMs) demonstrate impressive performance across various natural language processing tasks. Serving LLM inference for generating long contents, however, poses a challenge due to the enormous memory footprint of the transient state, known as the key-value (KV) cache, which scales with the sequence length and batch size. In this paper, we present InfiniGen, a novel KV cache management framework tailored for long-text generation, which synergistically works with modern offloading-based inference systems. InfiniGen leverages the key insight that a few important tokens that are essential for computing the subsequent attention layer in the Transformer can be speculated by performing a minimal rehearsal with the inputs of the current layer and part of the query weight and key cache of the subsequent layer. This allows us to prefetch only the essential KV cache entries (without fetching them all), thereby mitigating the fetch overhead from the host memory in offloading-based LLM serving systems. Our evaluation on several representative LLMs shows that InfiniGen improves the overall performance of a modern offloading-based system by up to 3.00x compared to prior KV cache management methods while offering substantially better model accuracy.","url_abs":"https://arxiv.org/abs/2406.19707v1","url_pdf":"https://arxiv.org/pdf/2406.19707v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"infinigen-efficient-generative-inference-of","repo_url":"https://github.com/snu-comparch/infinigen","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"management","task_name":"Management"},{"task_slug":"text-generation","task_name":"Text Generation"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2406.19707","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.19707"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/snu-comparch/infinigen","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_fixture":3,"ran_draft_wrong":1,"unverified":11},"by_repo_kind":{"listed":{"samples":15,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"30d7eec482ebf6b1","entry":"repeat_kv","repo":"snu-comparch/infinigen","repo_kind":"listed","path":"accuracy/src/modeling_llama_orig.py","file_url":"https://github.com/snu-comparch/infinigen/blob/HEAD/accuracy/src/modeling_llama_orig.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":2,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"30d7eec482ebf6b1"}},{"code_sha256_prefix":"9b4dff79d5e6102c","entry":"apply_rotary_pos_emb","repo":"snu-comparch/infinigen","repo_kind":"listed","path":"accuracy/src/modeling_llama_ours.py","file_url":"https://github.com/snu-comparch/infinigen/blob/HEAD/accuracy/src/modeling_llama_ours.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9b4dff79d5e6102c"}},{"code_sha256_prefix":"37fc3129c3d06e0c","entry":"apply_rotary_pos_emb","repo":"snu-comparch/infinigen","repo_kind":"listed","path":"accuracy/src/modeling_llama_orig.py","file_url":"https://github.com/snu-comparch/infinigen/blob/HEAD/accuracy/src/modeling_llama_orig.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"37fc3129c3d06e0c"}},{"code_sha256_prefix":"b99eea6376d1e212","entry":"rotate_half","repo":"snu-comparch/infinigen","repo_kind":"listed","path":"accuracy/src/modeling_llama_orig.py","file_url":"https://github.com/snu-comparch/infinigen/blob/HEAD/accuracy/src/modeling_llama_orig.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b99eea6376d1e212"}},{"code_sha256_prefix":"58fb93e90d4ca515","entry":"cut_indices","repo":"snu-comparch/infinigen","repo_kind":"listed","path":"speedup/flexgen/infinigen/pytorch_backend.py","file_url":"https://github.com/snu-comparch/infinigen/blob/HEAD/speedup/flexgen/infinigen/pytorch_backend.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"58fb93e90d4ca515"}},{"code_sha256_prefix":"66b0a9c4c8b9fac3","entry":"get_choice","repo":"snu-comparch/infinigen","repo_kind":"listed","path":"speedup/flexgen/infinigen/flex_opt.py","file_url":"https://github.com/snu-comparch/infinigen/blob/HEAD/speedup/flexgen/infinigen/flex_opt.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"66b0a9c4c8b9fac3"}},{"code_sha256_prefix":"aa4fd60bcb4735f7","entry":"get_filename","repo":"snu-comparch/infinigen","repo_kind":"listed","path":"speedup/flexgen/infinigen/flex_opt.py","file_url":"https://github.com/snu-comparch/infinigen/blob/HEAD/speedup/flexgen/infinigen/flex_opt.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"aa4fd60bcb4735f7"}},{"code_sha256_prefix":"68b8c44fbdc6b0bd","entry":"partial_weight_index_generation","repo":"snu-comparch/infinigen","repo_kind":"listed","path":"speedup/infinigen/infinigen/partial_weight_generation_controller.py","file_url":"https://github.com/snu-comparch/infinigen/blob/HEAD/speedup/infinigen/infinigen/partial_weight_generation_controller.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"68b8c44fbdc6b0bd"}},{"code_sha256_prefix":"60c756ef5ff6bb68","entry":"reform_hidden_states","repo":"snu-comparch/infinigen","repo_kind":"listed","path":"speedup/infinigen/infinigen/skewing_controller.py","file_url":"https://github.com/snu-comparch/infinigen/blob/HEAD/speedup/infinigen/infinigen/skewing_controller.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"60c756ef5ff6bb68"}},{"code_sha256_prefix":"dc50e1d3317233bc","entry":"select_kv","repo":"snu-comparch/infinigen","repo_kind":"listed","path":"speedup/infinigen/infinigen/kv_selection_controller.py","file_url":"https://github.com/snu-comparch/infinigen/blob/HEAD/speedup/infinigen/infinigen/kv_selection_controller.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"dc50e1d3317233bc"}},{"code_sha256_prefix":"c48042fd1baff5fa","entry":"set_partial_cache","repo":"snu-comparch/infinigen","repo_kind":"listed","path":"speedup/infinigen/infinigen/partial_weight_generation_controller.py","file_url":"https://github.com/snu-comparch/infinigen/blob/HEAD/speedup/infinigen/infinigen/partial_weight_generation_controller.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c48042fd1baff5fa"}},{"code_sha256_prefix":"8bd8cf4ddbf1f17b","entry":"set_partial_weight","repo":"snu-comparch/infinigen","repo_kind":"listed","path":"speedup/infinigen/infinigen/partial_weight_generation_controller.py","file_url":"https://github.com/snu-comparch/infinigen/blob/HEAD/speedup/infinigen/infinigen/partial_weight_generation_controller.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8bd8cf4ddbf1f17b"}},{"code_sha256_prefix":"7523546807dfeacd","entry":"skew","repo":"snu-comparch/infinigen","repo_kind":"listed","path":"speedup/infinigen/infinigen/skewing_controller.py","file_url":"https://github.com/snu-comparch/infinigen/blob/HEAD/speedup/infinigen/infinigen/skewing_controller.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7523546807dfeacd"}},{"code_sha256_prefix":"9e45f95449a51acf","entry":"speculate_attention","repo":"snu-comparch/infinigen","repo_kind":"listed","path":"speedup/infinigen/infinigen/kv_selection_controller.py","file_url":"https://github.com/snu-comparch/infinigen/blob/HEAD/speedup/infinigen/infinigen/kv_selection_controller.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9e45f95449a51acf"}},{"code_sha256_prefix":"65c813943048c026","entry":"weight_bias_concat","repo":"snu-comparch/infinigen","repo_kind":"listed","path":"speedup/infinigen/infinigen/skewing_controller.py","file_url":"https://github.com/snu-comparch/infinigen/blob/HEAD/speedup/infinigen/infinigen/skewing_controller.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"65c813943048c026"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}