{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/neo-saving-gpu-memory-crisis-with-cpu","title":"NEO: Saving GPU Memory Crisis with CPU Offloading for Online LLM Inference","arxiv_id":"2411.01142","date":"2024-11-02","proceeding":null,"authors":["Xuanlin Jiang","Yang Zhou","Shiyi Cao","Ion Stoica","Minlan Yu"],"abstract":"Online LLM inference powers many exciting applications such as intelligent chatbots and autonomous agents. Modern LLM inference engines widely rely on request batching to improve inference throughput, aiming to make it cost-efficient when running on expensive GPU accelerators. However, the limited GPU memory has largely limited the batch size achieved in practice, leaving significant GPU compute resources wasted. We present NEO, an online LLM inference system that offloads part of attention compute and KV cache states from the GPU to the local host CPU, effectively increasing the GPU batch size and thus inference throughput. To this end, NEO proposes asymmetric GPU-CPU pipelining and load-aware scheduling to balance GPU and CPU loads and fully utilize their compute and memory resources. We evaluate NEO on a wide range of workloads (i.e., code generation, text summarization), GPUs (i.e., T4, A10G, H100), and LLM models (i.e., 7B, 8B, 70B). NEO achieves up to 7.5$\\times$, 26%, and 14% higher throughput compared to GPU-only approach on T4, A10G, and H100 GPUs, respectively, while maintaining the same latency; with more powerful CPUs, NEO achieves up to 79.3% throughput gain on A10G GPU.","url_abs":"https://arxiv.org/abs/2411.01142v1","url_pdf":"https://arxiv.org/pdf/2411.01142v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"neo-saving-gpu-memory-crisis-with-cpu","repo_url":"https://github.com/MachineLearningSystem/25MLSYS-NEO","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":null,"task_name":"CPU"},{"task_slug":"code-generation","task_name":"Code Generation"},{"task_slug":null,"task_name":"GPU"},{"task_slug":"scheduling","task_name":"Scheduling"},{"task_slug":"text-summarization","task_name":"Text Summarization"}],"methods":[{"method_slug":"attention","method_name":"Attention"},{"method_slug":"softmax","method_name":"Softmax"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2411.01142","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2411.01142"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/MachineLearningSystem/25MLSYS-NEO","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"unverified":5},"by_repo_kind":{"listed":{"samples":5,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"a4f31d440ecccee6","entry":"draw_one_rl_diagram","repo":"MachineLearningSystem/25MLSYS-NEO","repo_kind":"listed","path":"evaluation/illustrator.py","file_url":"https://github.com/MachineLearningSystem/25MLSYS-NEO/blob/HEAD/evaluation/illustrator.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a4f31d440ecccee6"}},{"code_sha256_prefix":"f1eae6f23294c7bb","entry":"get_lat_avg","repo":"MachineLearningSystem/25MLSYS-NEO","repo_kind":"listed","path":"evaluation/illustrator.py","file_url":"https://github.com/MachineLearningSystem/25MLSYS-NEO/blob/HEAD/evaluation/illustrator.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f1eae6f23294c7bb"}},{"code_sha256_prefix":"4affe90a85f8702c","entry":"get_tp","repo":"MachineLearningSystem/25MLSYS-NEO","repo_kind":"listed","path":"evaluation/illustrator.py","file_url":"https://github.com/MachineLearningSystem/25MLSYS-NEO/blob/HEAD/evaluation/illustrator.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4affe90a85f8702c"}},{"code_sha256_prefix":"678f874464900355","entry":"prepare_mock_test","repo":"MachineLearningSystem/25MLSYS-NEO","repo_kind":"listed","path":"evaluation/benchmark.py","file_url":"https://github.com/MachineLearningSystem/25MLSYS-NEO/blob/HEAD/evaluation/benchmark.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"678f874464900355"}},{"code_sha256_prefix":"75de68bb7abb836e","entry":"prepare_real_test","repo":"MachineLearningSystem/25MLSYS-NEO","repo_kind":"listed","path":"evaluation/benchmark.py","file_url":"https://github.com/MachineLearningSystem/25MLSYS-NEO/blob/HEAD/evaluation/benchmark.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"75de68bb7abb836e"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}