{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/llm-int8-8-bit-matrix-multiplication-for","title":"LLM.int8(): 8-bit Matrix Multiplication for Transformers at Scale","arxiv_id":"2208.07339","date":"2022-08-15","proceeding":null,"authors":["Tim Dettmers","Mike Lewis","Younes Belkada","Luke Zettlemoyer"],"abstract":"Large language models have been widely adopted but require significant GPU memory for inference. We develop a procedure for Int8 matrix multiplication for feed-forward and attention projection layers in transformers, which cut the memory needed for inference by half while retaining full precision performance. With our method, a 175B parameter 16/32-bit checkpoint can be loaded, converted to Int8, and used immediately without performance degradation. This is made possible by understanding and working around properties of highly systematic emergent features in transformer language models that dominate attention and transformer predictive performance. To cope with these features, we develop a two-part quantization procedure, LLM.int8(). We first use vector-wise quantization with separate normalization constants for each inner product in the matrix multiplication, to quantize most of the features. However, for the emergent outliers, we also include a new mixed-precision decomposition scheme, which isolates the outlier feature dimensions into a 16-bit matrix multiplication while still more than 99.9% of values are multiplied in 8-bit. Using LLM.int8(), we show empirically it is possible to perform inference in LLMs with up to 175B parameters without any performance degradation. This result makes such models much more accessible, for example making it possible to use OPT-175B/BLOOM on a single server with consumer GPUs. We open-source our software.","url_abs":"https://arxiv.org/abs/2208.07339v2","url_pdf":"https://arxiv.org/pdf/2208.07339v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"llm-int8-8-bit-matrix-multiplication-for","repo_url":"https://github.com/timdettmers/bitsandbytes","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"llm-int8-8-bit-matrix-multiplication-for","repo_url":"https://github.com/alextmallen/adaptive-retrieval","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"llm-int8-8-bit-matrix-multiplication-for","repo_url":"https://github.com/huggingface/transformers-bloom-inference","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"llm-int8-8-bit-matrix-multiplication-for","repo_url":"https://github.com/kohjingyu/fromage","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":null,"task_name":"GPU"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"linguistic-acceptability","task_name":"Linguistic Acceptability"},{"task_slug":"natural-language-inference","task_name":"Natural Language Inference"},{"task_slug":"quantization","task_name":"Quantization"},{"task_slug":"semantic-textual-similarity","task_name":"Semantic Textual Similarity"},{"task_slug":"sentiment-analysis","task_name":"Sentiment Analysis"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/language-modelling-on-c4","task":"Language Modelling","dataset":"C4","model":"Zeropoint LLM.int8 13B (vector-wise + decomp)","rank_in_archive_order":2,"of":9,"metrics":{"Perplexity":"12.45"},"uses_additional_data":false},{"leaderboard":"/sota/language-modelling-on-c4","task":"Language Modelling","dataset":"C4","model":"LLM.float32 6.7B","rank_in_archive_order":5,"of":9,"metrics":{"Perplexity":"13.3"},"uses_additional_data":false},{"leaderboard":"/sota/language-modelling-on-c4","task":"Language Modelling","dataset":"C4","model":"LLM.float32 2.7B","rank_in_archive_order":6,"of":9,"metrics":{"Perplexity":"14.43"},"uses_additional_data":false},{"leaderboard":"/sota/language-modelling-on-c4","task":"Language Modelling","dataset":"C4","model":"LLM.float32 1.3B","rank_in_archive_order":9,"of":9,"metrics":{"Perplexity":"15.91"},"uses_additional_data":false},{"leaderboard":"/sota/linguistic-acceptability-on-cola","task":"Linguistic Acceptability","dataset":"CoLA","model":"RoBERTa-large 355M (MLP quantized vector-wise, fine-tuned)","rank_in_archive_order":17,"of":43,"metrics":{"Accuracy":"68.6%"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-multinli","task":"Natural Language Inference","dataset":"MultiNLI","model":"RoBERTa-large 355M (MLP quantized vector-wise, fine-tuned)","rank_in_archive_order":11,"of":67,"metrics":{"Matched":"90.2"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-qnli","task":"Natural Language Inference","dataset":"QNLI","model":"RoBERTa-large 355M (MLP quantized vector-wise, fine-tuned)","rank_in_archive_order":13,"of":43,"metrics":{"Accuracy":"94.7%"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-rte","task":"Natural Language Inference","dataset":"RTE","model":"RoBERTa-large 355M (MLP quantized vector-wise, fine-tuned)","rank_in_archive_order":25,"of":90,"metrics":{"Accuracy":"85.4%"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-mrpc","task":"Semantic Textual Similarity","dataset":"MRPC","model":"RoBERTa-large 355M (MLP quantized vector-wise, fine-tuned)","rank_in_archive_order":7,"of":45,"metrics":{"Accuracy":"91.0%"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-sts-benchmark","task":"Semantic Textual Similarity","dataset":"STS Benchmark","model":"RoBERTa-large 355M (MLP quantized vector-wise, fine-tuned)","rank_in_archive_order":9,"of":66,"metrics":{"Pearson Correlation":"0.919"},"uses_additional_data":false},{"leaderboard":"/sota/sentiment-analysis-on-sst-2-binary","task":"Sentiment Analysis","dataset":"SST-2 Binary classification","model":"RoBERTa-large 355M (MLP quantized vector-wise, fine-tuned)","rank_in_archive_order":16,"of":87,"metrics":{"Accuracy":"96.4"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2208.07339","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2208.07339"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kohjingyu/fromage","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/timdettmers/bitsandbytes","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/huggingface/transformers-bloom-inference","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/alextmallen/adaptive-retrieval","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":2,"unverified":3},"by_repo_kind":{"official":{"samples":1,"ran":0,"repositories":1},"listed":{"samples":4,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"5f7de07e75040659","entry":"get_benchmark_results","repo":"huggingface/transformers-bloom-inference","repo_kind":"listed","path":"inference_server/benchmark.py","file_url":"https://github.com/huggingface/transformers-bloom-inference/blob/HEAD/inference_server/benchmark.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5f7de07e75040659"}},{"code_sha256_prefix":"23648dfbe89d08f8","entry":"get_model_path","repo":"huggingface/transformers-bloom-inference","repo_kind":"listed","path":"inference_server/models/ds_inference.py","file_url":"https://github.com/huggingface/transformers-bloom-inference/blob/HEAD/inference_server/models/ds_inference.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"23648dfbe89d08f8"}},{"code_sha256_prefix":"9442182e69129d23","entry":"get_hf_model_class","repo":"huggingface/transformers-bloom-inference","repo_kind":"listed","path":"inference_server/models/model.py","file_url":"https://github.com/huggingface/transformers-bloom-inference/blob/HEAD/inference_server/models/model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9442182e69129d23"}},{"code_sha256_prefix":"646f7cd7192b9ea8","entry":"int8_linear_matmul","repo":"timdettmers/bitsandbytes","repo_kind":"official","path":"bitsandbytes/functional.py","file_url":"https://github.com/timdettmers/bitsandbytes/blob/HEAD/bitsandbytes/functional.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"646f7cd7192b9ea8"}},{"code_sha256_prefix":"835634e64b6b277d","entry":"load_tokenizer","repo":"huggingface/transformers-bloom-inference","repo_kind":"listed","path":"inference_server/models/model.py","file_url":"https://github.com/huggingface/transformers-bloom-inference/blob/HEAD/inference_server/models/model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"835634e64b6b277d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}