{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/a-systematic-study-of-cross-layer-kv-sharing","title":"A Systematic Study of Cross-Layer KV Sharing for Efficient LLM Inference","arxiv_id":"2410.14442","date":"2024-10-18","proceeding":null,"authors":["You Wu","HaoYi Wu","Kewei Tu"],"abstract":"Recently, sharing key-value (KV) cache across layers has been found effective in efficient inference of large language models (LLMs). To systematically investigate different techniques of cross-layer KV sharing, we propose a unified framework that covers several recent methods and their novel variants. We conduct comprehensive experiments on all the configurations of the framework, evaluating their generation throughput and performance in language modeling and downstream tasks. We find that when reducing the size of the KV cache by 2x, most configurations can achieve competitive performance to and higher throughput than standard transformers, but when further reducing the size of the KV cache, pairing queries of all layers with KVs of upper layers can better maintain performance, although it also introduces additional training cost and prefilling latency. We hope that this work will help users choose the appropriate approach according to their requirements and facilitate research on the acceleration of LLM inference.","url_abs":"https://arxiv.org/abs/2410.14442v1","url_pdf":"https://arxiv.org/pdf/2410.14442v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"a-systematic-study-of-cross-layer-kv-sharing","repo_url":"https://github.com/whyNLP/LCKV","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2410.14442","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.14442"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/whyNLP/LCKV","reach":{"status":"ok"}}],"summary":{"ran":4,"ran_draft_wrong":3,"unverified":2},"by_repo_kind":{"official":{"samples":9,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":9,"samples":[{"code_sha256_prefix":"259b212edddd369f","entry":"clear_chat_history","repo":"whyNLP/LCKV","repo_kind":"official","path":"run_chat.py","file_url":"https://github.com/whyNLP/LCKV/blob/HEAD/run_chat.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"259b212edddd369f"}},{"code_sha256_prefix":"b28eddb4feb75e0f","entry":"create_default_filename","repo":"whyNLP/LCKV","repo_kind":"official","path":"run_chat.py","file_url":"https://github.com/whyNLP/LCKV/blob/HEAD/run_chat.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b28eddb4feb75e0f"}},{"code_sha256_prefix":"a766a95b9fd5ee9c","entry":"prepare_ctrl_input","repo":"whyNLP/LCKV","repo_kind":"official","path":"run_generation.py","file_url":"https://github.com/whyNLP/LCKV/blob/HEAD/run_generation.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a766a95b9fd5ee9c"}},{"code_sha256_prefix":"b7bf53f6b6b11a4b","entry":"prepare_xlm_input","repo":"whyNLP/LCKV","repo_kind":"official","path":"run_generation.py","file_url":"https://github.com/whyNLP/LCKV/blob/HEAD/run_generation.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b7bf53f6b6b11a4b"}},{"code_sha256_prefix":"80844fa7cfa6e8d8","entry":"prepare_xlnet_input","repo":"whyNLP/LCKV","repo_kind":"official","path":"run_generation.py","file_url":"https://github.com/whyNLP/LCKV/blob/HEAD/run_generation.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"80844fa7cfa6e8d8"}},{"code_sha256_prefix":"66dd156d52d85aae","entry":"save_chat","repo":"whyNLP/LCKV","repo_kind":"official","path":"run_chat.py","file_url":"https://github.com/whyNLP/LCKV/blob/HEAD/run_chat.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"66dd156d52d85aae"}},{"code_sha256_prefix":"25136b330739fb24","entry":"tinyllama_get_lr","repo":"whyNLP/LCKV","repo_kind":"official","path":"run_clm.py","file_url":"https://github.com/whyNLP/LCKV/blob/HEAD/run_clm.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"25136b330739fb24"}},{"code_sha256_prefix":"92eac4dd828162e7","entry":"flash_attention_forward","repo":"whyNLP/LCKV","repo_kind":"official","path":"models/utils.py","file_url":"https://github.com/whyNLP/LCKV/blob/HEAD/models/utils.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"92eac4dd828162e7"}},{"code_sha256_prefix":"daa55e6ce848f98f","entry":"liger_rotary","repo":"whyNLP/LCKV","repo_kind":"official","path":"models/kernel.py","file_url":"https://github.com/whyNLP/LCKV/blob/HEAD/models/kernel.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"daa55e6ce848f98f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}