{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/identify-critical-kv-cache-in-llm-inference","title":"Identify Critical KV Cache in LLM Inference from an Output Perturbation Perspective","arxiv_id":"2502.03805","date":"2025-02-06","proceeding":null,"authors":["Yuan Feng","Junlin Lv","Yukun Cao","Xike Xie","S Kevin Zhou"],"abstract":"Large language models have revolutionized natural language processing but face significant challenges of high storage and runtime costs, due to the transformer architecture's reliance on self-attention, particularly the large Key-Value (KV) cache for long-sequence inference. Recent efforts to reduce KV cache size by pruning less critical entries based on attention weights remain empirical and lack formal grounding. This paper presents a formal study on identifying critical KV cache entries by analyzing attention output perturbation. Our analysis reveals that, beyond attention weights, the value states within KV entries and pretrained parameter matrices are also crucial. Based on this, we propose a perturbation-constrained selection algorithm that optimizes the worst-case output perturbation to identify critical entries. Evaluations on the Needle-in-a-Haystack test and Longbench benchmark show our algorithm enhances state-of-the-art cache eviction methods. Further empirical analysis confirms that our algorithm achieves lower output perturbations in over 92% attention heads in Llama model, thereby providing a significant improvement over existing methods.","url_abs":"https://arxiv.org/abs/2502.03805v1","url_pdf":"https://arxiv.org/pdf/2502.03805v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"identify-critical-kv-cache-in-llm-inference","repo_url":"https://github.com/NVIDIA/kvpress","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"identify-critical-kv-cache-in-llm-inference","repo_url":"https://github.com/ffy0/adakv","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[],"methods":[{"method_slug":"attention","method_name":"Attention"},{"method_slug":"llama","method_name":"LLaMA"},{"method_slug":"pruning","method_name":"Pruning"},{"method_slug":"softmax","method_name":"Softmax"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2502.03805","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2502.03805"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/FFY0/DefensiveKV","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ffy0/adakv","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/NVIDIA/kvpress","reach":null}],"summary":{"ran":2,"ran_fixture":1,"unverified":8},"by_repo_kind":{"official":{"samples":4,"ran":0,"repositories":1},"found_in_text":{"samples":7,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"653f147f7dedf90f","entry":"DynamicCacheSplitHeadFlatten","repo":"FFY0/DefensiveKV","repo_kind":"found_in_text","path":"kvpress/presses/criticalkv_press.py","file_url":"https://github.com/FFY0/DefensiveKV/blob/HEAD/kvpress/presses/criticalkv_press.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"653f147f7dedf90f"}},{"code_sha256_prefix":"346206adb8bf5312","entry":"MetaData","repo":"FFY0/DefensiveKV","repo_kind":"found_in_text","path":"kvpress/presses/criticalkv_press.py","file_url":"https://github.com/FFY0/DefensiveKV/blob/HEAD/kvpress/presses/criticalkv_press.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"346206adb8bf5312"}},{"code_sha256_prefix":"24eafaba79154752","entry":"get_prerope_query_states","repo":"FFY0/DefensiveKV","repo_kind":"found_in_text","path":"kvpress/presses/criticalkv_press.py","file_url":"https://github.com/FFY0/DefensiveKV/blob/HEAD/kvpress/presses/criticalkv_press.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"24eafaba79154752"}},{"code_sha256_prefix":"7d325a6d956c95ae","entry":"BasePress","repo":"FFY0/DefensiveKV","repo_kind":"found_in_text","path":"kvpress/presses/criticalkv_press.py","file_url":"https://github.com/FFY0/DefensiveKV/blob/HEAD/kvpress/presses/criticalkv_press.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7d325a6d956c95ae"}},{"code_sha256_prefix":"1bb41fbe40f0cb94","entry":"BasePress","repo":"NVIDIA/kvpress","repo_kind":"official","path":"kvpress/presses/criticalkv_press.py","file_url":"https://github.com/NVIDIA/kvpress/blob/HEAD/kvpress/presses/criticalkv_press.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1bb41fbe40f0cb94"}},{"code_sha256_prefix":"bbb411fe894a3c42","entry":"CriticalKVPress","repo":"FFY0/DefensiveKV","repo_kind":"found_in_text","path":"kvpress/presses/criticalkv_press.py","file_url":"https://github.com/FFY0/DefensiveKV/blob/HEAD/kvpress/presses/criticalkv_press.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bbb411fe894a3c42"}},{"code_sha256_prefix":"05f4c9404e12f85a","entry":"CriticalKVPress","repo":"NVIDIA/kvpress","repo_kind":"official","path":"kvpress/presses/criticalkv_press.py","file_url":"https://github.com/NVIDIA/kvpress/blob/HEAD/kvpress/presses/criticalkv_press.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"05f4c9404e12f85a"}},{"code_sha256_prefix":"77ac43dd592848b4","entry":"ExpectedAttentionPress","repo":"FFY0/DefensiveKV","repo_kind":"found_in_text","path":"kvpress/presses/criticalkv_press.py","file_url":"https://github.com/FFY0/DefensiveKV/blob/HEAD/kvpress/presses/criticalkv_press.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"77ac43dd592848b4"}},{"code_sha256_prefix":"34a4bcd44bd02b5b","entry":"ExpectedAttentionPress","repo":"NVIDIA/kvpress","repo_kind":"official","path":"kvpress/presses/criticalkv_press.py","file_url":"https://github.com/NVIDIA/kvpress/blob/HEAD/kvpress/presses/criticalkv_press.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"34a4bcd44bd02b5b"}},{"code_sha256_prefix":"02fb35ba14327c55","entry":"ScorerPress","repo":"FFY0/DefensiveKV","repo_kind":"found_in_text","path":"kvpress/presses/criticalkv_press.py","file_url":"https://github.com/FFY0/DefensiveKV/blob/HEAD/kvpress/presses/criticalkv_press.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"02fb35ba14327c55"}},{"code_sha256_prefix":"bc245f692797739a","entry":"ScorerPress","repo":"NVIDIA/kvpress","repo_kind":"official","path":"kvpress/presses/criticalkv_press.py","file_url":"https://github.com/NVIDIA/kvpress/blob/HEAD/kvpress/presses/criticalkv_press.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bc245f692797739a"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}