{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/examining-post-training-quantization-for","title":"Examining Post-Training Quantization for Mixture-of-Experts: A Benchmark","arxiv_id":"2406.08155","date":"2024-06-12","proceeding":null,"authors":["Pingzhi Li","Xiaolong Jin","Yu Cheng","Tianlong Chen"],"abstract":"Large Language Models~(LLMs) have become foundational in the realm of natural language processing, demonstrating performance improvements as model sizes increase. The Mixture-of-Experts~(MoE) approach offers a promising way to scale LLMs more efficiently by using fewer computational FLOPs through sparse activation. However, it suffers from significant memory overheads, necessitating model compression techniques. Post-training quantization, a popular method for model compression, proves less effective when directly applied to MoE models due to MoE's overlooked inherent sparsity. This paper explores several MoE structure-aware quantization heuristics, ranging from coarse to fine granularity, from MoE block to individual linear weight. Our investigations reveal critical principles: different MoE structures (i.e., blocks, experts, linear layers) require varying numbers of weight bits for effective and efficient quantization. Conclusions are supported by extensive benchmarking across two representative MoE models and six tasks. We further introduce novel enhancements to more accurately identify the most critical weights in MoE quantization that necessitate higher bit allocations, including the linear weight outlier scorer and MoE block scorer. Additionally, subsequent experiments validate our findings in the context of both weight and activation quantization.","url_abs":"https://arxiv.org/abs/2406.08155v1","url_pdf":"https://arxiv.org/pdf/2406.08155v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"examining-post-training-quantization-for","repo_url":"https://github.com/unites-lab/moe-quantization","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"benchmarking","task_name":"Benchmarking"},{"task_slug":"mixture-of-experts","task_name":"Mixture-of-Experts"},{"task_slug":"model-compression","task_name":"Model Compression"},{"task_slug":"quantization","task_name":"Quantization"}],"methods":[{"method_slug":"moe","method_name":"MoE"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2406.08155","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.08155"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/unites-lab/moe-quantization","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_fixture":1,"unverified":5},"by_repo_kind":{"official":{"samples":6,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"41c57e55d21c4d96","entry":"quantize","repo":"unites-lab/moe-quantization","repo_kind":"official","path":"auto_gptq/quantization/quantizer.py","file_url":"https://github.com/unites-lab/moe-quantization/blob/HEAD/auto_gptq/quantization/quantizer.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"41c57e55d21c4d96"}},{"code_sha256_prefix":"0c44615fb82626c5","entry":"collect_mixtral_block_output","repo":"unites-lab/moe-quantization","repo_kind":"official","path":"dump-mixtral-block-output.py","file_url":"https://github.com/unites-lab/moe-quantization/blob/HEAD/dump-mixtral-block-output.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0c44615fb82626c5"}},{"code_sha256_prefix":"e1edb8f5a7b1441c","entry":"collect_mixtral_ffn_predictor_train_data","repo":"unites-lab/moe-quantization","repo_kind":"official","path":"collect-mixtral-predictor-data.py","file_url":"https://github.com/unites-lab/moe-quantization/blob/HEAD/collect-mixtral-predictor-data.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e1edb8f5a7b1441c"}},{"code_sha256_prefix":"754ae82011aa091a","entry":"collect_mixtral_ffn_with_residual_predictor_train_data","repo":"unites-lab/moe-quantization","repo_kind":"official","path":"collect-mixtral-predictor-data.py","file_url":"https://github.com/unites-lab/moe-quantization/blob/HEAD/collect-mixtral-predictor-data.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"754ae82011aa091a"}},{"code_sha256_prefix":"b05a67dcc919fa71","entry":"collect_mixtral_predictor_test_data","repo":"unites-lab/moe-quantization","repo_kind":"official","path":"collect-mixtral-predictor-data.py","file_url":"https://github.com/unites-lab/moe-quantization/blob/HEAD/collect-mixtral-predictor-data.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b05a67dcc919fa71"}},{"code_sha256_prefix":"219f020120bf3e16","entry":"get_wikitext2","repo":"unites-lab/moe-quantization","repo_kind":"official","path":"dump_mixtral_routing_distribution.py","file_url":"https://github.com/unites-lab/moe-quantization/blob/HEAD/dump_mixtral_routing_distribution.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"219f020120bf3e16"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}