{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/compressed-context-memory-for-online-language","title":"Compressed Context Memory For Online Language Model Interaction","arxiv_id":"2312.03414","date":"2023-12-06","proceeding":null,"authors":["Jang-Hyun Kim","Junyoung Yeom","Sangdoo Yun","Hyun Oh Song"],"abstract":"This paper presents a context key/value compression method for Transformer language models in online scenarios, where the context continually expands. As the context lengthens, the attention process demands increasing memory and computations, which in turn reduces the throughput of the language model. To address this challenge, we propose a compressed context memory system that continually compresses the accumulating attention key/value pairs into a compact memory space, facilitating language model inference in a limited memory space of computing environments. Our compression process involves integrating a lightweight conditional LoRA into the language model's forward pass during inference, without the need for fine-tuning the model's entire set of weights. We achieve efficient training by modeling the recursive compression process as a single parallelized forward computation. Through evaluations on conversation, personalization, and multi-task learning, we demonstrate that our approach achieves the performance level of a full context model with $5\\times$ smaller context memory size. We further demonstrate the applicability of our approach in a streaming setting with an unlimited context length, outperforming the sliding window approach. Codes are available at https://github.com/snu-mllab/context-memory.","url_abs":"https://arxiv.org/abs/2312.03414v2","url_pdf":"https://arxiv.org/pdf/2312.03414v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"compressed-context-memory-for-online-language","repo_url":"https://github.com/snu-mllab/context-memory","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"multi-task-learning","task_name":"Multi-Task Learning"},{"task_slug":"model","task_name":"model"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"set","method_name":"SET"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2312.03414","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.03414"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/snu-mllab/context-memory","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_fixture":3,"ran":5,"ran_draft_wrong":1,"unverified":4},"by_repo_kind":{"official":{"samples":13,"ran":9,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"30d7eec482ebf6b1","entry":"repeat_kv","repo":"snu-mllab/context-memory","repo_kind":"official","path":"src/arch/ccm_mistral.py","file_url":"https://github.com/snu-mllab/context-memory/blob/HEAD/src/arch/ccm_mistral.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":2,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"30d7eec482ebf6b1"}},{"code_sha256_prefix":"d61c483a3c2b3156","entry":"apply_rotary_pos_emb","repo":"snu-mllab/context-memory","repo_kind":"official","path":"src/arch/ccm_mistral.py","file_url":"https://github.com/snu-mllab/context-memory/blob/HEAD/src/arch/ccm_mistral.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d61c483a3c2b3156"}},{"code_sha256_prefix":"5695c3c3dfeb3e3f","entry":"first_mismatch","repo":"snu-mllab/context-memory","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/snu-mllab/context-memory/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5695c3c3dfeb3e3f"}},{"code_sha256_prefix":"14c6ecb20ee83621","entry":"map_config","repo":"snu-mllab/context-memory","repo_kind":"official","path":"path_config.py","file_url":"https://github.com/snu-mllab/context-memory/blob/HEAD/path_config.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"14c6ecb20ee83621"}},{"code_sha256_prefix":"c0d66785e485f1f9","entry":"model_path","repo":"snu-mllab/context-memory","repo_kind":"official","path":"path_config.py","file_url":"https://github.com/snu-mllab/context-memory/blob/HEAD/path_config.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c0d66785e485f1f9"}},{"code_sha256_prefix":"b99eea6376d1e212","entry":"rotate_half","repo":"snu-mllab/context-memory","repo_kind":"official","path":"src/arch/ccm_mistral.py","file_url":"https://github.com/snu-mllab/context-memory/blob/HEAD/src/arch/ccm_mistral.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b99eea6376d1e212"}},{"code_sha256_prefix":"d73f2cdc8a6da456","entry":"simple_basename","repo":"snu-mllab/context-memory","repo_kind":"official","path":"src/arguments.py","file_url":"https://github.com/snu-mllab/context-memory/blob/HEAD/src/arguments.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d73f2cdc8a6da456"}},{"code_sha256_prefix":"8f1a49e9d8b29d83","entry":"transpose","repo":"snu-mllab/context-memory","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/snu-mllab/context-memory/blob/HEAD/src/utils.py","link_basis":"plan_row","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8f1a49e9d8b29d83"}},{"code_sha256_prefix":"ec53310b0e0ad465","entry":"update_position_ids","repo":"snu-mllab/context-memory","repo_kind":"official","path":"src/arch/ccm_llama.py","file_url":"https://github.com/snu-mllab/context-memory/blob/HEAD/src/arch/ccm_llama.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ec53310b0e0ad465"}},{"code_sha256_prefix":"e1c94faaa4993ea9","entry":"extract_comp_results","repo":"snu-mllab/context-memory","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/snu-mllab/context-memory/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e1c94faaa4993ea9"}},{"code_sha256_prefix":"eb6f2ebdeb4cd0ca","entry":"get_conditional_lora","repo":"snu-mllab/context-memory","repo_kind":"official","path":"src/model.py","file_url":"https://github.com/snu-mllab/context-memory/blob/HEAD/src/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"eb6f2ebdeb4cd0ca"}},{"code_sha256_prefix":"6cc12d12ed256c22","entry":"get_lora","repo":"snu-mllab/context-memory","repo_kind":"official","path":"src/model.py","file_url":"https://github.com/snu-mllab/context-memory/blob/HEAD/src/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6cc12d12ed256c22"}},{"code_sha256_prefix":"ba83723330c0338e","entry":"get_traininable_state_dict","repo":"snu-mllab/context-memory","repo_kind":"official","path":"src/model.py","file_url":"https://github.com/snu-mllab/context-memory/blob/HEAD/src/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ba83723330c0338e"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}