{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/model-tells-you-what-to-discard-adaptive-kv","title":"Model Tells You What to Discard: Adaptive KV Cache Compression for LLMs","arxiv_id":"2310.01801","date":"2023-10-03","proceeding":null,"authors":["Suyu Ge","Yunan Zhang","Liyuan Liu","Minjia Zhang","Jiawei Han","Jianfeng Gao"],"abstract":"In this study, we introduce adaptive KV cache compression, a plug-and-play method that reduces the memory footprint of generative inference for Large Language Models (LLMs). Different from the conventional KV cache that retains key and value vectors for all context tokens, we conduct targeted profiling to discern the intrinsic structure of attention modules. Based on the recognized structure, we then construct the KV cache in an adaptive manner: evicting long-range contexts on attention heads emphasizing local contexts, discarding non-special tokens on attention heads centered on special tokens, and only employing the standard KV cache for attention heads that broadly attend to all tokens. Moreover, with the lightweight attention profiling used to guide the construction of the adaptive KV cache, FastGen can be deployed without resource-intensive fine-tuning or re-training. In our experiments across various asks, FastGen demonstrates substantial reduction on GPU memory consumption with negligible generation quality loss. We will release our code and the compatible CUDA kernel for reproducibility.","url_abs":"https://arxiv.org/abs/2310.01801v4","url_pdf":"https://arxiv.org/pdf/2310.01801v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"model-tells-you-what-to-discard-adaptive-kv","repo_url":"https://github.com/machilusz/fastgen","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"none","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"model-tells-you-what-to-discard-adaptive-kv","repo_url":"https://github.com/answerdotai/cold-compress","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":null,"task_name":"GPU"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2310.01801","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.01801"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/answerdotai/cold-compress","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/machilusz/fastgen","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_violates":1,"ran_fixture":4,"unverified":4},"by_repo_kind":{"listed":{"samples":8,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":1,"samples":[{"code_sha256_prefix":"472d821ef38be37d","entry":"create_window_attention_mask","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"472d821ef38be37d"}},{"code_sha256_prefix":"5564321cfa1d2c94","entry":"dequantize_tensor","repo":"answerdotai/cold-compress","repo_kind":"listed","path":"cache.py","file_url":"https://github.com/answerdotai/cold-compress/blob/HEAD/cache.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"5564321cfa1d2c94"}},{"code_sha256_prefix":"f7e4d6a116aaae11","entry":"pack_low_bit_tensor","repo":"answerdotai/cold-compress","repo_kind":"listed","path":"cache.py","file_url":"https://github.com/answerdotai/cold-compress/blob/HEAD/cache.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"f7e4d6a116aaae11"}},{"code_sha256_prefix":"8d5153e794aa8b15","entry":"quantize_tensor","repo":"answerdotai/cold-compress","repo_kind":"listed","path":"cache.py","file_url":"https://github.com/answerdotai/cold-compress/blob/HEAD/cache.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"8d5153e794aa8b15"}},{"code_sha256_prefix":"910108db225dd9fe","entry":"unpack_low_bit_tensor","repo":"answerdotai/cold-compress","repo_kind":"listed","path":"cache.py","file_url":"https://github.com/answerdotai/cold-compress/blob/HEAD/cache.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"910108db225dd9fe"}},{"code_sha256_prefix":"77d2d58362eb96eb","entry":"KVCache","repo":"answerdotai/cold-compress","repo_kind":"listed","path":"cache.py","file_url":"https://github.com/answerdotai/cold-compress/blob/HEAD/cache.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"77d2d58362eb96eb"}},{"code_sha256_prefix":"8e720cf08e93134e","entry":"KVCacheHeadSpecific","repo":"answerdotai/cold-compress","repo_kind":"listed","path":"cache.py","file_url":"https://github.com/answerdotai/cold-compress/blob/HEAD/cache.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"8e720cf08e93134e"}},{"code_sha256_prefix":"b74628786f18097a","entry":"KVCacheHeavyHitter","repo":"answerdotai/cold-compress","repo_kind":"listed","path":"cache.py","file_url":"https://github.com/answerdotai/cold-compress/blob/HEAD/cache.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"b74628786f18097a"}},{"code_sha256_prefix":"bc3c2262d92a1205","entry":"KVCacheHybrid","repo":"answerdotai/cold-compress","repo_kind":"listed","path":"cache.py","file_url":"https://github.com/answerdotai/cold-compress/blob/HEAD/cache.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"bc3c2262d92a1205"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}