{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/ape-faster-and-longer-context-augmented","title":"APE: Faster and Longer Context-Augmented Generation via Adaptive Parallel Encoding","arxiv_id":"2502.05431","date":"2025-02-08","proceeding":null,"authors":["Xinyu Yang","Tianqi Chen","Beidi Chen"],"abstract":"Context-augmented generation (CAG) techniques, including RAG and ICL, require the efficient combination of multiple contexts to generate responses to user queries. Directly inputting these contexts as a sequence introduces a considerable computational burden by re-encoding the combined selection of contexts for every request. To address this, we explore the promising potential of parallel encoding to independently pre-compute and cache each context's KV states. This approach enables the direct loading of cached states during inference while accommodating more contexts through position reuse across contexts. However, due to misalignments in attention distribution, directly applying parallel encoding results in a significant performance drop. To enable effective and efficient CAG, we propose Adaptive Parallel Encoding ($\\textbf{APE}$), which brings shared prefix, attention temperature, and scaling factor to align the distribution of parallel encoding with sequential encoding. Results on RAG and ICL tasks demonstrate that APE can preserve 98% and 93% sequential encoding performance using the same inputs while outperforming parallel encoding by 3.6% and 7.9%, respectively. It also scales to many-shot CAG, effectively encoding hundreds of contexts in parallel. Efficiency evaluation shows that APE can achieve an end-to-end 4.5$\\times$ speedup by reducing 28$\\times$ prefilling time for a 128K-length context.","url_abs":"https://arxiv.org/abs/2502.05431v2","url_pdf":"https://arxiv.org/pdf/2502.05431v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"ape-faster-and-longer-context-augmented","repo_url":"https://github.com/infini-ai-lab/ape","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":null}],"tasks":[{"task_slug":"rag","task_name":"RAG"}],"methods":[{"method_slug":"align","method_name":"ALIGN"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bart","method_name":"BART"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"cag","method_name":"CAG"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"heatmap","method_name":"Heatmap"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"rag","method_name":"RAG"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2502.05431","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2502.05431"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/infini-ai-lab/ape","reach":null}],"summary":{"ran_violates":1,"ran_draft_wrong":3,"ran_fixture":1},"by_repo_kind":{"official":{"samples":5,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":5,"samples":[{"code_sha256_prefix":"685c6e6865a76b21","entry":"build_prefix","repo":"infini-ai-lab/ape","repo_kind":"official","path":"demo_ape.py","file_url":"https://github.com/infini-ai-lab/ape/blob/HEAD/demo_ape.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"685c6e6865a76b21"}},{"code_sha256_prefix":"376fbfafe97d0322","entry":"combine_output_files","repo":"infini-ai-lab/ape","repo_kind":"official","path":"experiments/ChatRAG-Bench/evaluation/run_generation_parallel.py","file_url":"https://github.com/infini-ai-lab/ape/blob/HEAD/experiments/ChatRAG-Bench/evaluation/run_generation_parallel.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"376fbfafe97d0322"}},{"code_sha256_prefix":"3473ac9115dcb86b","entry":"gemma_forward","repo":"infini-ai-lab/ape","repo_kind":"official","path":"ape/ape_gemma.py","file_url":"https://github.com/infini-ai-lab/ape/blob/HEAD/ape/ape_gemma.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3473ac9115dcb86b"}},{"code_sha256_prefix":"622253aec5a5363c","entry":"parse_args","repo":"infini-ai-lab/ape","repo_kind":"official","path":"demo_ape.py","file_url":"https://github.com/infini-ai-lab/ape/blob/HEAD/demo_ape.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"622253aec5a5363c"}},{"code_sha256_prefix":"8d0a0843f2ce4634","entry":"write_to_separate_file","repo":"infini-ai-lab/ape","repo_kind":"official","path":"experiments/ChatRAG-Bench/evaluation/run_generation_parallel.py","file_url":"https://github.com/infini-ai-lab/ape/blob/HEAD/experiments/ChatRAG-Bench/evaluation/run_generation_parallel.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8d0a0843f2ce4634"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}