{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2603-00364","title":"Distribution-Aware Companding Quantization of Large Language Models","arxiv_id":"2603.00364","date":"2026-02-27","proceeding":null,"authors":["Athul Radhakrishnan","Siddhant Mohan","Mahima Sachdeva"],"abstract":"Large language models such as GPT and Llama are trained with a next-token prediction loss. In this work, we suggest that training language models to predict multiple future tokens at once results in higher sample efficiency. More specifically, at each position in the training corpus, we ask the model to predict the following n tokens using n independent output heads, operating on top of a shared model trunk. Considering multi-token prediction as an auxiliary training task, we measure improved downstream capabilities with no overhead in training time for both code and natural language models. The method is increasingly useful for larger model sizes and keeps its appeal when training for multiple epochs. Gains are especially pronounced on generative benchmarks like coding, where our models consistently outperform strong baselines by several percentage points. Our 13B parameter models solves 12 % more problems on HumanEval and 17 % more on MBPP than comparable next-token models. Experiments on small algorithmic tasks demonstrate that multi-token prediction is favorable for the development of induction heads and algorithmic reasoning capabilities. As an additional benefit, models trained with 4-token prediction are up to 3X times faster at inference, even with large batch sizes.","url_abs":"https://arxiv.org/abs/2603.00364","url_pdf":"https://arxiv.org/pdf/2603.00364","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2603.00364","atlas_url":"https://app.syntology.ai/?focus=2603.00364","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2603.00364"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/Athul-R/dist-quant","reach":null}],"summary":{"ran_draft_wrong":4},"by_repo_kind":{"found_in_text":{"samples":4,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"7bd86abfc867f753","entry":"awq_vs_dacq_logistic_torch","repo":"Athul-R/dist-quant","repo_kind":"found_in_text","path":"awq_v_dacq.py","file_url":"https://github.com/Athul-R/dist-quant/blob/HEAD/awq_v_dacq.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7bd86abfc867f753"}},{"code_sha256_prefix":"97dab6c8eb0b412a","entry":"logistic_cdf_torch","repo":"Athul-R/dist-quant","repo_kind":"found_in_text","path":"awq_v_dacq.py","file_url":"https://github.com/Athul-R/dist-quant/blob/HEAD/awq_v_dacq.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"97dab6c8eb0b412a"}},{"code_sha256_prefix":"4b9b695675629bf2","entry":"logistic_fit_torch","repo":"Athul-R/dist-quant","repo_kind":"found_in_text","path":"awq_v_dacq.py","file_url":"https://github.com/Athul-R/dist-quant/blob/HEAD/awq_v_dacq.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4b9b695675629bf2"}},{"code_sha256_prefix":"709c500df128de51","entry":"logistic_icdf_torch","repo":"Athul-R/dist-quant","repo_kind":"found_in_text","path":"awq_v_dacq.py","file_url":"https://github.com/Athul-R/dist-quant/blob/HEAD/awq_v_dacq.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"709c500df128de51"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.CL","source":"arxiv_2026.jsonl"},"syntology_extracted_results":null}