{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/multi-token-prediction-needs-registers","title":"Multi-Token Prediction Needs Registers","arxiv_id":"2505.10518","date":"2025-05-15","proceeding":null,"authors":["Anastasios Gerontopoulos","Spyros Gidaris","Nikos Komodakis"],"abstract":"Multi-token prediction has emerged as a promising objective for improving language model pretraining, but its benefits have not consistently generalized to other settings such as fine-tuning. In this paper, we propose MuToR, a simple and effective approach to multi-token prediction that interleaves learnable register tokens into the input sequence, each tasked with predicting future targets. Compared to existing methods, MuToR offers several key advantages: it introduces only a negligible number of additional parameters, requires no architectural changes--ensuring compatibility with off-the-shelf pretrained language models--and remains aligned with the next-token pretraining objective, making it especially well-suited for supervised fine-tuning. Moreover, it naturally supports scalable prediction horizons. We demonstrate the effectiveness and versatility of MuToR across a range of use cases, including supervised fine-tuning, parameter-efficient fine-tuning (PEFT), and pretraining, on challenging generative tasks in both language and vision domains. Our code will be available at: https://github.com/nasosger/MuToR.","url_abs":"https://arxiv.org/abs/2505.10518v1","url_pdf":"https://arxiv.org/pdf/2505.10518v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"multi-token-prediction-needs-registers","repo_url":"https://github.com/nasosger/mutor","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"none","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"prediction","task_name":"Prediction"},{"task_slug":"parameter-efficient-fine-tuning","task_name":"parameter-efficient fine-tuning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2505.10518","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.10518"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/nasosger/mutor","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_fixture":2,"ran_draft_wrong":5,"ran":1,"unverified":12},"by_repo_kind":{"official":{"samples":20,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"30d7eec482ebf6b1","entry":"repeat_kv","repo":"nasosger/mutor","repo_kind":"official","path":"language_modeling/src/models/llama_mutor.py","file_url":"https://github.com/nasosger/mutor/blob/HEAD/language_modeling/src/models/llama_mutor.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":2,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"30d7eec482ebf6b1"}},{"code_sha256_prefix":"3c76e52815c5401d","entry":"repeat_kv","repo":"nasosger/mutor","repo_kind":"official","path":"language_modeling/src/models/gemma_mutor.py","file_url":"https://github.com/nasosger/mutor/blob/HEAD/language_modeling/src/models/gemma_mutor.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":2,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3c76e52815c5401d"}},{"code_sha256_prefix":"bac65c3dafaec040","entry":"apply_rotary_pos_emb","repo":"nasosger/mutor","repo_kind":"official","path":"language_modeling/src/models/llama_mutor.py","file_url":"https://github.com/nasosger/mutor/blob/HEAD/language_modeling/src/models/llama_mutor.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bac65c3dafaec040"}},{"code_sha256_prefix":"583539efd6fd01fb","entry":"apply_rotary_pos_emb","repo":"nasosger/mutor","repo_kind":"official","path":"language_modeling/src/models/gemma_mutor.py","file_url":"https://github.com/nasosger/mutor/blob/HEAD/language_modeling/src/models/gemma_mutor.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"583539efd6fd01fb"}},{"code_sha256_prefix":"855a4d8c22777aac","entry":"last_boxed_only","repo":"nasosger/mutor","repo_kind":"official","path":"language_modeling/src/eval/math_utils.py","file_url":"https://github.com/nasosger/mutor/blob/HEAD/language_modeling/src/eval/math_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"855a4d8c22777aac"}},{"code_sha256_prefix":"0b14c648516c38a7","entry":"last_boxed_only_string","repo":"nasosger/mutor","repo_kind":"official","path":"language_modeling/src/eval/math_utils.py","file_url":"https://github.com/nasosger/mutor/blob/HEAD/language_modeling/src/eval/math_utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0b14c648516c38a7"}},{"code_sha256_prefix":"b99eea6376d1e212","entry":"rotate_half","repo":"nasosger/mutor","repo_kind":"official","path":"language_modeling/src/models/llama_mutor.py","file_url":"https://github.com/nasosger/mutor/blob/HEAD/language_modeling/src/models/llama_mutor.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b99eea6376d1e212"}},{"code_sha256_prefix":"e03d53ba9d4f9ae5","entry":"rotate_half","repo":"nasosger/mutor","repo_kind":"official","path":"language_modeling/src/models/gemma_mutor.py","file_url":"https://github.com/nasosger/mutor/blob/HEAD/language_modeling/src/models/gemma_mutor.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e03d53ba9d4f9ae5"}},{"code_sha256_prefix":"09a73cf0b65ef0df","entry":"create_dataloaders","repo":"nasosger/mutor","repo_kind":"official","path":"language_modeling/src/dataloaders_mutor.py","file_url":"https://github.com/nasosger/mutor/blob/HEAD/language_modeling/src/dataloaders_mutor.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"09a73cf0b65ef0df"}},{"code_sha256_prefix":"0e4c30e9e6691a25","entry":"extract_answer","repo":"nasosger/mutor","repo_kind":"official","path":"language_modeling/src/eval/evaluate_gsm8k.py","file_url":"https://github.com/nasosger/mutor/blob/HEAD/language_modeling/src/eval/evaluate_gsm8k.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0e4c30e9e6691a25"}},{"code_sha256_prefix":"1c971191751f6f79","entry":"extract_boxed_answer","repo":"nasosger/mutor","repo_kind":"official","path":"language_modeling/src/eval/evaluate_gsm8k.py","file_url":"https://github.com/nasosger/mutor/blob/HEAD/language_modeling/src/eval/evaluate_gsm8k.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1c971191751f6f79"}},{"code_sha256_prefix":"f78a5e52e103712f","entry":"generate_greedy_with_kv_cache","repo":"nasosger/mutor","repo_kind":"official","path":"language_modeling/src/eval/generation_utils.py","file_url":"https://github.com/nasosger/mutor/blob/HEAD/language_modeling/src/eval/generation_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f78a5e52e103712f"}},{"code_sha256_prefix":"61aa6f4de0349de0","entry":"get_grouped_params","repo":"nasosger/mutor","repo_kind":"official","path":"language_modeling/src/finetune.py","file_url":"https://github.com/nasosger/mutor/blob/HEAD/language_modeling/src/finetune.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"61aa6f4de0349de0"}},{"code_sha256_prefix":"105d7363a921e299","entry":"json_loads","repo":"nasosger/mutor","repo_kind":"official","path":"language_modeling/src/eval/evaluate_math500.py","file_url":"https://github.com/nasosger/mutor/blob/HEAD/language_modeling/src/eval/evaluate_math500.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"105d7363a921e299"}},{"code_sha256_prefix":"49122c3bfdc8df43","entry":"log_trainable_params","repo":"nasosger/mutor","repo_kind":"official","path":"language_modeling/src/finetune.py","file_url":"https://github.com/nasosger/mutor/blob/HEAD/language_modeling/src/finetune.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"49122c3bfdc8df43"}},{"code_sha256_prefix":"45a49a8bffc94e4c","entry":"maybe_remove_comma","repo":"nasosger/mutor","repo_kind":"official","path":"language_modeling/src/eval/evaluate_gsm8k.py","file_url":"https://github.com/nasosger/mutor/blob/HEAD/language_modeling/src/eval/evaluate_gsm8k.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"45a49a8bffc94e4c"}},{"code_sha256_prefix":"0bbf876ced018c83","entry":"only_until_first_boxed_from_tokens","repo":"nasosger/mutor","repo_kind":"official","path":"language_modeling/src/eval/math_utils.py","file_url":"https://github.com/nasosger/mutor/blob/HEAD/language_modeling/src/eval/math_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0bbf876ced018c83"}},{"code_sha256_prefix":"9294b12a2eccb090","entry":"prepare_datasets","repo":"nasosger/mutor","repo_kind":"official","path":"language_modeling/src/dataloaders_mutor.py","file_url":"https://github.com/nasosger/mutor/blob/HEAD/language_modeling/src/dataloaders_mutor.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9294b12a2eccb090"}},{"code_sha256_prefix":"a0b5557b356e0c05","entry":"setup_logging","repo":"nasosger/mutor","repo_kind":"official","path":"language_modeling/src/finetune.py","file_url":"https://github.com/nasosger/mutor/blob/HEAD/language_modeling/src/finetune.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a0b5557b356e0c05"}},{"code_sha256_prefix":"380e3b6083d1d00a","entry":"update_config","repo":"nasosger/mutor","repo_kind":"official","path":"language_modeling/src/args_mutor.py","file_url":"https://github.com/nasosger/mutor/blob/HEAD/language_modeling/src/args_mutor.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"380e3b6083d1d00a"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}