{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/leanquant-accurate-large-language-model","title":"LeanQuant: Accurate Large Language Model Quantization with Loss-Error-Aware Grid","arxiv_id":"2407.10032","date":"2024-07-14","proceeding":null,"authors":["Tianyi Zhang","Anshumali Shrivastava"],"abstract":"Large language models (LLMs) have numerous applications across various domains, but their high computational and memory demands pose significant deployment challenges. Weight quantization is an effective technique for reducing the decoding latency and memory requirements of LLMs. Existing approaches primarily aim to maintain the quality of quantized models by preserving outliers in input features, but they still suffer significant quality loss at lower bit widths. Our approach builds on Optimal Brain Quantization (OBQ), an iterative weight-update-based quantization framework. We identify a key limitation of OBQ, specifically that its uniform quantization grid is suboptimal for maintaining model quality, as it introduces large errors to the task loss. To address this, we propose LeanQuant, which learns a loss-error-aware quantization grid by leveraging the inverse diagonal Hessian. Extensive empirical evaluations demonstrate that LeanQuant is both efficient and accurate; it can quantize a 70-billion-parameter model in 6 hours using a single 32GB GPU and performs favorably compared to competitive baselines in the 4-bit, 3-bit, and 2-bit regions.","url_abs":"https://arxiv.org/abs/2407.10032v1","url_pdf":"https://arxiv.org/pdf/2407.10032v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[],"tasks":[{"task_slug":null,"task_name":"GPU"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"large-language-model","task_name":"Large Language Model"},{"task_slug":"quantization","task_name":"Quantization"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2407.10032","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.10032"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/LeanModels/LeanQuant","reach":{"status":"ok"}}],"summary":{"ran_draft_wrong":1,"ran":1,"ran_fixture":1,"unverified":4},"by_repo_kind":{"found_in_text":{"samples":7,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":7,"samples":[{"code_sha256_prefix":"a9e7f2cdf016b88b","entry":"find_layers","repo":"LeanModels/LeanQuant","repo_kind":"found_in_text","path":"modelutils.py","file_url":"https://github.com/LeanModels/LeanQuant/blob/HEAD/modelutils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a9e7f2cdf016b88b"}},{"code_sha256_prefix":"f810b1ddd55b5375","entry":"get_ptb","repo":"LeanModels/LeanQuant","repo_kind":"found_in_text","path":"datautils.py","file_url":"https://github.com/LeanModels/LeanQuant/blob/HEAD/datautils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f810b1ddd55b5375"}},{"code_sha256_prefix":"41c57e55d21c4d96","entry":"quantize","repo":"LeanModels/LeanQuant","repo_kind":"found_in_text","path":"quant.py","file_url":"https://github.com/LeanModels/LeanQuant/blob/HEAD/quant.py","link_basis":"plan_row","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"41c57e55d21c4d96"}},{"code_sha256_prefix":"86ac8f1de9c0550b","entry":"get_c4","repo":"LeanModels/LeanQuant","repo_kind":"found_in_text","path":"datautils.py","file_url":"https://github.com/LeanModels/LeanQuant/blob/HEAD/datautils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"86ac8f1de9c0550b"}},{"code_sha256_prefix":"9c16d58cb9509ae5","entry":"get_llama","repo":"LeanModels/LeanQuant","repo_kind":"found_in_text","path":"llama.py","file_url":"https://github.com/LeanModels/LeanQuant/blob/HEAD/llama.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9c16d58cb9509ae5"}},{"code_sha256_prefix":"3bbe14523c98ed57","entry":"get_wikitext2","repo":"LeanModels/LeanQuant","repo_kind":"found_in_text","path":"datautils.py","file_url":"https://github.com/LeanModels/LeanQuant/blob/HEAD/datautils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3bbe14523c98ed57"}},{"code_sha256_prefix":"340b2f9ae0efeee0","entry":"kmeans_fit","repo":"LeanModels/LeanQuant","repo_kind":"found_in_text","path":"lean_quantizer.py","file_url":"https://github.com/LeanModels/LeanQuant/blob/HEAD/lean_quantizer.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"340b2f9ae0efeee0"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}