{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/hexgen-generative-inference-of-foundation","title":"HexGen: Generative Inference of Large Language Model over Heterogeneous Environment","arxiv_id":"2311.11514","date":"2023-11-20","proceeding":null,"authors":["Youhe Jiang","Ran Yan","Xiaozhe Yao","Yang Zhou","Beidi Chen","Binhang Yuan"],"abstract":"Serving generative inference of the large language model is a crucial component of contemporary AI applications. This paper focuses on deploying such services in a heterogeneous and cross-datacenter setting to mitigate the substantial inference costs typically associated with a single centralized datacenter. Towards this end, we propose HexGen, a flexible distributed inference engine that uniquely supports the asymmetric partition of generative inference computations over both tensor model parallelism and pipeline parallelism and allows for effective deployment across diverse GPUs interconnected by a fully heterogeneous network. We further propose a sophisticated scheduling algorithm grounded in constrained optimization that can adaptively assign asymmetric inference computation across the GPUs to fulfill inference requests while maintaining acceptable latency levels. We conduct an extensive evaluation to verify the efficiency of HexGen by serving the state-of-the-art Llama-2 (70B) model. The results suggest that HexGen can choose to achieve up to 2.3 times lower latency deadlines or tolerate up to 4 times more request rates compared with the homogeneous baseline given the same budget.","url_abs":"https://arxiv.org/abs/2311.11514v3","url_pdf":"https://arxiv.org/pdf/2311.11514v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"links_only","authors_date_abstract":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license), from the Kaggle arXiv metadata snapshot of 2026-09-12"},"code_links":[{"paper_slug":"hexgen-generative-inference-of-foundation","repo_url":"https://github.com/relaxed-system-lab/hexgen","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2311.11514","atlas_url":"https://app.syntology.ai/?focus=2311.11514","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.11514"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/relaxed-system-lab/hexgen","reach":null}],"summary":{"ran_draft_wrong":3,"unverified":7},"by_repo_kind":{"official":{"samples":10,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"0d939562cce3288d","entry":"chunk_batch","repo":"relaxed-system-lab/hexgen","repo_kind":"official","path":"hexgen/hexgen_core/heterogeneous_pipeline.py","file_url":"https://github.com/relaxed-system-lab/hexgen/blob/HEAD/hexgen/hexgen_core/heterogeneous_pipeline.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0d939562cce3288d"}},{"code_sha256_prefix":"be32a1cb7f68aa17","entry":"forward_step_func","repo":"Relaxed-System-Lab/HexGen","repo_kind":"official","path":"hexgen/llama/llama_inference.py","file_url":"https://github.com/Relaxed-System-Lab/HexGen/blob/HEAD/hexgen/llama/llama_inference.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"be32a1cb7f68aa17"}},{"code_sha256_prefix":"85b5684d7d73dde9","entry":"unwrap_model","repo":"relaxed-system-lab/hexgen","repo_kind":"official","path":"hexgen/hexgen_core/heterogeneous_pipeline.py","file_url":"https://github.com/relaxed-system-lab/hexgen/blob/HEAD/hexgen/hexgen_core/heterogeneous_pipeline.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"85b5684d7d73dde9"}},{"code_sha256_prefix":"c7bc94829fa993b8","entry":"PipelineParallel","repo":"relaxed-system-lab/hexgen","repo_kind":"official","path":"hexgen/hexgen_core/heterogeneous_pipeline.py","file_url":"https://github.com/relaxed-system-lab/hexgen/blob/HEAD/hexgen/hexgen_core/heterogeneous_pipeline.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c7bc94829fa993b8"}},{"code_sha256_prefix":"132c20e5030631e6","entry":"delete_ddp_backward_hook","repo":"relaxed-system-lab/hexgen","repo_kind":"official","path":"hexgen/hexgen_core/heterogeneous_pipeline.py","file_url":"https://github.com/relaxed-system-lab/hexgen/blob/HEAD/hexgen/hexgen_core/heterogeneous_pipeline.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"132c20e5030631e6"}},{"code_sha256_prefix":"d4881cd6a9d551d8","entry":"delete_fsdp_post_backward_hook","repo":"relaxed-system-lab/hexgen","repo_kind":"official","path":"hexgen/hexgen_core/heterogeneous_pipeline.py","file_url":"https://github.com/relaxed-system-lab/hexgen/blob/HEAD/hexgen/hexgen_core/heterogeneous_pipeline.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d4881cd6a9d551d8"}},{"code_sha256_prefix":"f21a8acd941c3235","entry":"forward_delete_backward_hook","repo":"relaxed-system-lab/hexgen","repo_kind":"official","path":"hexgen/hexgen_core/heterogeneous_pipeline.py","file_url":"https://github.com/relaxed-system-lab/hexgen/blob/HEAD/hexgen/hexgen_core/heterogeneous_pipeline.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f21a8acd941c3235"}},{"code_sha256_prefix":"e38a8d0be6c626f6","entry":"post_pipeline_forward","repo":"relaxed-system-lab/hexgen","repo_kind":"official","path":"hexgen/hexgen_core/heterogeneous_pipeline.py","file_url":"https://github.com/relaxed-system-lab/hexgen/blob/HEAD/hexgen/hexgen_core/heterogeneous_pipeline.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e38a8d0be6c626f6"}},{"code_sha256_prefix":"6fc0d1e65a75823d","entry":"pre_pipeline_forward","repo":"relaxed-system-lab/hexgen","repo_kind":"official","path":"hexgen/hexgen_core/heterogeneous_pipeline.py","file_url":"https://github.com/relaxed-system-lab/hexgen/blob/HEAD/hexgen/hexgen_core/heterogeneous_pipeline.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6fc0d1e65a75823d"}},{"code_sha256_prefix":"bab49f78b2154c6f","entry":"rewrite_fsdp_forward_no_post_backward","repo":"relaxed-system-lab/hexgen","repo_kind":"official","path":"hexgen/hexgen_core/heterogeneous_pipeline.py","file_url":"https://github.com/relaxed-system-lab/hexgen/blob/HEAD/hexgen/hexgen_core/heterogeneous_pipeline.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bab49f78b2154c6f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}