{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/partition-generative-modeling-masked-modeling","title":"Partition Generative Modeling: Masked Modeling Without Masks","arxiv_id":"2505.18883","date":"2025-05-24","proceeding":null,"authors":["Justin Deschenaux","Lan Tran","Caglar Gulcehre"],"abstract":"We introduce ``Partition Generative Models'' (PGMs), a novel approach to masked generative modeling (MGMs), particularly effective for masked diffusion language modeling (MDLMs). PGM divides tokens into two distinct groups and employs sparse attention patterns to prevent cross-group information exchange. Hence, the model is trained to predict tokens in one group based solely on information from the other group. This partitioning strategy eliminates the need for MASK tokens entirely. While traditional MGMs inefficiently process MASK tokens during generation, PGMs achieve greater computational efficiency by operating exclusively on unmasked tokens. Our experiments on OpenWebText with a context length of 1024 tokens demonstrate that PGMs deliver at least 5x improvements in both latency and throughput compared to MDLM when using the same number of sampling steps, while generating samples with better generative perplexity than MDLM. Finally, we show that PGMs can be distilled with Self-Distillation Through Time (SDTT), a method originally devised for MDLM, in order to achieve further inference gains.","url_abs":"https://arxiv.org/abs/2505.18883v1","url_pdf":"https://arxiv.org/pdf/2505.18883v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"partition-generative-modeling-masked-modeling","repo_url":"https://github.com/kuleshov-group/mdlm","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"computational-efficiency","task_name":"Computational Efficiency"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"}],"methods":[{"method_slug":"attention","method_name":"Attention"},{"method_slug":"diffusion","method_name":"Diffusion"},{"method_slug":"pgm","method_name":"PGM"},{"method_slug":"softmax","method_name":"Softmax"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2505.18883","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.18883"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kuleshov-group/mdlm","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_fixture":2,"ran":5,"ran_honours":1,"unverified":6},"by_repo_kind":{"official":{"samples":14,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"fbbc6722b74bda27","entry":"bias_dropout_add_scale","repo":"kuleshov-group/mdlm","repo_kind":"official","path":"models/autoregressive.py","file_url":"https://github.com/kuleshov-group/mdlm/blob/HEAD/models/autoregressive.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fbbc6722b74bda27"}},{"code_sha256_prefix":"ee894c881614c3ec","entry":"bias_dropout_add_scale_fused_train","repo":"kuleshov-group/mdlm","repo_kind":"official","path":"models/autoregressive.py","file_url":"https://github.com/kuleshov-group/mdlm/blob/HEAD/models/autoregressive.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ee894c881614c3ec"}},{"code_sha256_prefix":"dec1de00023a923b","entry":"fsspec_exists","repo":"kuleshov-group/mdlm","repo_kind":"official","path":"utils.py","file_url":"https://github.com/kuleshov-group/mdlm/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"dec1de00023a923b"}},{"code_sha256_prefix":"06d2d8e50f91481f","entry":"get_bias_dropout_add_scale","repo":"kuleshov-group/mdlm","repo_kind":"official","path":"models/autoregressive.py","file_url":"https://github.com/kuleshov-group/mdlm/blob/HEAD/models/autoregressive.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"06d2d8e50f91481f"}},{"code_sha256_prefix":"576e1b9b9bcae8ff","entry":"lm1b_detokenizer","repo":"kuleshov-group/mdlm","repo_kind":"official","path":"dataloader.py","file_url":"https://github.com/kuleshov-group/mdlm/blob/HEAD/dataloader.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"576e1b9b9bcae8ff"}},{"code_sha256_prefix":"78b555034273f3c7","entry":"modulate","repo":"kuleshov-group/mdlm","repo_kind":"official","path":"models/dit.py","file_url":"https://github.com/kuleshov-group/mdlm/blob/HEAD/models/dit.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"78b555034273f3c7"}},{"code_sha256_prefix":"9f7fbdbe7ca280e0","entry":"ptb_detokenizer","repo":"kuleshov-group/mdlm","repo_kind":"official","path":"dataloader.py","file_url":"https://github.com/kuleshov-group/mdlm/blob/HEAD/dataloader.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9f7fbdbe7ca280e0"}},{"code_sha256_prefix":"46c10627206a7795","entry":"wt_detokenizer","repo":"kuleshov-group/mdlm","repo_kind":"official","path":"dataloader.py","file_url":"https://github.com/kuleshov-group/mdlm/blob/HEAD/dataloader.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"46c10627206a7795"}},{"code_sha256_prefix":"9b9814cca50339d7","entry":"binary_discretization","repo":"kuleshov-group/mdlm","repo_kind":"official","path":"noise_schedule.py","file_url":"https://github.com/kuleshov-group/mdlm/blob/HEAD/noise_schedule.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9b9814cca50339d7"}},{"code_sha256_prefix":"38b9682eb110eda2","entry":"cross_entropy","repo":"kuleshov-group/mdlm","repo_kind":"official","path":"models/dimamba.py","file_url":"https://github.com/kuleshov-group/mdlm/blob/HEAD/models/dimamba.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"38b9682eb110eda2"}},{"code_sha256_prefix":"db0136eb80a50383","entry":"fsspec_listdir","repo":"kuleshov-group/mdlm","repo_kind":"official","path":"utils.py","file_url":"https://github.com/kuleshov-group/mdlm/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"db0136eb80a50383"}},{"code_sha256_prefix":"60dbd956810fafb6","entry":"get_logger","repo":"kuleshov-group/mdlm","repo_kind":"official","path":"utils.py","file_url":"https://github.com/kuleshov-group/mdlm/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"60dbd956810fafb6"}},{"code_sha256_prefix":"fe6314378d2def00","entry":"get_noise","repo":"kuleshov-group/mdlm","repo_kind":"official","path":"noise_schedule.py","file_url":"https://github.com/kuleshov-group/mdlm/blob/HEAD/noise_schedule.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fe6314378d2def00"}},{"code_sha256_prefix":"fdbb5a9a25cff18f","entry":"weighted_cross_entropy","repo":"kuleshov-group/mdlm","repo_kind":"official","path":"models/dimamba.py","file_url":"https://github.com/kuleshov-group/mdlm/blob/HEAD/models/dimamba.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fdbb5a9a25cff18f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}