{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/onebit-towards-extremely-low-bit-large","title":"OneBit: Towards Extremely Low-bit Large Language Models","arxiv_id":"2402.11295","date":"2024-02-17","proceeding":null,"authors":["Yuzhuang Xu","Xu Han","Zonghan Yang","Shuo Wang","Qingfu Zhu","Zhiyuan Liu","Weidong Liu","Wanxiang Che"],"abstract":"Model quantification uses low bit-width values to represent the weight matrices of existing models to be quantized, which is a promising approach to reduce both storage and computational overheads of deploying highly anticipated LLMs. However, current quantization methods suffer severe performance degradation when the bit-width is extremely reduced, and thus focus on utilizing 4-bit or 8-bit values to quantize models. This paper boldly quantizes the weight matrices of LLMs to 1-bit, paving the way for the extremely low bit-width deployment of LLMs. For this target, we introduce a 1-bit model compressing framework named OneBit, including a novel 1-bit parameter representation method to better quantize LLMs as well as an effective parameter initialization method based on matrix decomposition to improve the convergence speed of the quantization framework. Sufficient experimental results indicate that OneBit achieves good performance (at least 81% of the non-quantized performance on LLaMA models) with robust training processes when only using 1-bit weight matrices.","url_abs":"https://arxiv.org/abs/2402.11295v6","url_pdf":"https://arxiv.org/pdf/2402.11295v6.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"onebit-towards-extremely-low-bit-large","repo_url":"https://github.com/xuyuzhuang11/onebit","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"jax","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"quantization","task_name":"Quantization"}],"methods":[{"method_slug":"focus","method_name":"Focus"},{"method_slug":"llama","method_name":"LLaMA"},{"method_slug":"speed","method_name":"SPEED"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2402.11295","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.11295"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/xuyuzhuang11/OneBit","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/xuyuzhuang11/onebit","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":3,"ran":3,"unverified":6},"by_repo_kind":{"official":{"samples":12,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"9cfdc209c76fc7a5","entry":"find_all_linear_modules","repo":"xuyuzhuang11/onebit","repo_kind":"official","path":"llama_factory/llamafactory/core.py","file_url":"https://github.com/xuyuzhuang11/onebit/blob/HEAD/llama_factory/llamafactory/core.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9cfdc209c76fc7a5"}},{"code_sha256_prefix":"904cd61df2fe8fb9","entry":"find_layers","repo":"xuyuzhuang11/OneBit","repo_kind":"official","path":"evaluation/lm_eval/models_utils.py","file_url":"https://github.com/xuyuzhuang11/OneBit/blob/HEAD/evaluation/lm_eval/models_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"904cd61df2fe8fb9"}},{"code_sha256_prefix":"ea06eaae4fc1eaf0","entry":"hash_args","repo":"xuyuzhuang11/OneBit","repo_kind":"official","path":"evaluation/lm_eval/base.py","file_url":"https://github.com/xuyuzhuang11/OneBit/blob/HEAD/evaluation/lm_eval/base.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ea06eaae4fc1eaf0"}},{"code_sha256_prefix":"4df9168e0a0c38eb","entry":"make_disjoint_window","repo":"xuyuzhuang11/OneBit","repo_kind":"official","path":"evaluation/lm_eval/models_utils.py","file_url":"https://github.com/xuyuzhuang11/OneBit/blob/HEAD/evaluation/lm_eval/models_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4df9168e0a0c38eb"}},{"code_sha256_prefix":"c12b55ef635e9947","entry":"pattern_match","repo":"xuyuzhuang11/OneBit","repo_kind":"official","path":"evaluation/lm_eval/evaluator.py","file_url":"https://github.com/xuyuzhuang11/OneBit/blob/HEAD/evaluation/lm_eval/evaluator.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c12b55ef635e9947"}},{"code_sha256_prefix":"4e53fbbdfa160f7d","entry":"truncate_number","repo":"xuyuzhuang11/OneBit","repo_kind":"official","path":"evaluation/lm_eval/models_utils.py","file_url":"https://github.com/xuyuzhuang11/OneBit/blob/HEAD/evaluation/lm_eval/models_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4e53fbbdfa160f7d"}},{"code_sha256_prefix":"b176efb26f72e507","entry":"get_architectures_from_config_class","repo":"xuyuzhuang11/OneBit","repo_kind":"official","path":"transformers/utils/create_dummy_models.py","file_url":"https://github.com/xuyuzhuang11/OneBit/blob/HEAD/transformers/utils/create_dummy_models.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b176efb26f72e507"}},{"code_sha256_prefix":"3d999b95009c7537","entry":"get_config_class_from_processor_class","repo":"xuyuzhuang11/OneBit","repo_kind":"official","path":"transformers/utils/create_dummy_models.py","file_url":"https://github.com/xuyuzhuang11/OneBit/blob/HEAD/transformers/utils/create_dummy_models.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3d999b95009c7537"}},{"code_sha256_prefix":"ffa31cd3568da2b9","entry":"get_pile","repo":"xuyuzhuang11/OneBit","repo_kind":"official","path":"evaluation/lm_eval/datautils.py","file_url":"https://github.com/xuyuzhuang11/OneBit/blob/HEAD/evaluation/lm_eval/datautils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ffa31cd3568da2b9"}},{"code_sha256_prefix":"0c0b81b081919d8e","entry":"get_processor_types_from_config_class","repo":"xuyuzhuang11/OneBit","repo_kind":"official","path":"transformers/utils/create_dummy_models.py","file_url":"https://github.com/xuyuzhuang11/OneBit/blob/HEAD/transformers/utils/create_dummy_models.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0c0b81b081919d8e"}},{"code_sha256_prefix":"90dfde049522b716","entry":"get_ptb","repo":"xuyuzhuang11/OneBit","repo_kind":"official","path":"evaluation/lm_eval/datautils.py","file_url":"https://github.com/xuyuzhuang11/OneBit/blob/HEAD/evaluation/lm_eval/datautils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"90dfde049522b716"}},{"code_sha256_prefix":"27d7c058388f491d","entry":"get_wikitext2","repo":"xuyuzhuang11/OneBit","repo_kind":"official","path":"evaluation/lm_eval/datautils.py","file_url":"https://github.com/xuyuzhuang11/OneBit/blob/HEAD/evaluation/lm_eval/datautils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"27d7c058388f491d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}