{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/maskgwm-a-generalizable-driving-world-model","title":"MaskGWM: A Generalizable Driving World Model with Video Mask Reconstruction","arxiv_id":"2502.11663","date":"2025-02-17","proceeding":"CVPR 2025 1","authors":["Jingcheng Ni","Yuxin Guo","Yichen Liu","Rui Chen","Lewei Lu","Zehuan Wu"],"abstract":"World models that forecast environmental changes from actions are vital for autonomous driving models with strong generalization. The prevailing driving world model mainly build on video prediction model. Although these models can produce high-fidelity video sequences with advanced diffusion-based generator, they are constrained by their predictive duration and overall generalization capabilities. In this paper, we explore to solve this problem by combining generation loss with MAE-style feature-level context learning. In particular, we instantiate this target with three key design: (1) A more scalable Diffusion Transformer (DiT) structure trained with extra mask construction task. (2) we devise diffusion-related mask tokens to deal with the fuzzy relations between mask reconstruction and generative diffusion process. (3) we extend mask construction task to spatial-temporal domain by utilizing row-wise mask for shifted self-attention rather than masked self-attention in MAE. Then, we adopt a row-wise cross-view module to align with this mask design. Based on above improvement, we propose MaskGWM: a Generalizable driving World Model embodied with Video Mask reconstruction. Our model contains two variants: MaskGWM-long, focusing on long-horizon prediction, and MaskGWM-mview, dedicated to multi-view generation. Comprehensive experiments on standard benchmarks validate the effectiveness of the proposed method, which contain normal validation of Nuscene dataset, long-horizon rollout of OpenDV-2K dataset and zero-shot validation of Waymo dataset. Quantitative metrics on these datasets show our method notably improving state-of-the-art driving world model.","url_abs":"https://arxiv.org/abs/2502.11663v1","url_pdf":"https://arxiv.org/pdf/2502.11663v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"maskgwm-a-generalizable-driving-world-model","repo_url":"https://github.com/sensetime-fvg/opendwm","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"2k","task_name":"2k"},{"task_slug":"autonomous-driving","task_name":"Autonomous Driving"},{"task_slug":"task-2","task_name":"Task 2"},{"task_slug":"video-prediction","task_name":"Video Prediction"}],"methods":[{"method_slug":"adopt","method_name":"ADOPT"},{"method_slug":"align","method_name":"ALIGN"},{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"diffusion","method_name":"Diffusion"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"mae","method_name":"MAE"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2502.11663","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2502.11663"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sensetime-fvg/opendwm","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/SenseTime-FVG/OpenDWM","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":1,"ran_draft_wrong":1,"ran_fixture":1,"ran_honours":1,"unverified":4},"by_repo_kind":{"official":{"samples":7,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":1,"samples":[{"code_sha256_prefix":"d95c624cec119af3","entry":"Mlp","repo":"SenseTime-FVG/OpenDWM","repo_kind":"official","path":"src/dwm/models/mask_layers.py","file_url":"https://github.com/SenseTime-FVG/OpenDWM/blob/HEAD/src/dwm/models/mask_layers.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d95c624cec119af3"}},{"code_sha256_prefix":"edcc588e588a39da","entry":"_ntuple","repo":"SenseTime-FVG/OpenDWM","repo_kind":"official","path":"src/dwm/models/mask_layers.py","file_url":"https://github.com/SenseTime-FVG/OpenDWM/blob/HEAD/src/dwm/models/mask_layers.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"edcc588e588a39da"}},{"code_sha256_prefix":"eb05305555a13cb2","entry":"get_layernorm","repo":"SenseTime-FVG/OpenDWM","repo_kind":"official","path":"src/dwm/models/mask_layers.py","file_url":"https://github.com/SenseTime-FVG/OpenDWM/blob/HEAD/src/dwm/models/mask_layers.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"DEP_MISSING","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"eb05305555a13cb2"}},{"code_sha256_prefix":"a5e3d9618aac5205","entry":"t2i_modulate","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"a5e3d9618aac5205"}},{"code_sha256_prefix":"d78be5ac90fc43de","entry":"MaskController","repo":"SenseTime-FVG/OpenDWM","repo_kind":"official","path":"src/dwm/models/mask_layers.py","file_url":"https://github.com/SenseTime-FVG/OpenDWM/blob/HEAD/src/dwm/models/mask_layers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d78be5ac90fc43de"}},{"code_sha256_prefix":"9266208320f3adcf","entry":"MaskPatchEmbed","repo":"SenseTime-FVG/OpenDWM","repo_kind":"official","path":"src/dwm/models/mask_layers.py","file_url":"https://github.com/SenseTime-FVG/OpenDWM/blob/HEAD/src/dwm/models/mask_layers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9266208320f3adcf"}},{"code_sha256_prefix":"a51979aca82e80a4","entry":"STDiT3Block","repo":"SenseTime-FVG/OpenDWM","repo_kind":"official","path":"src/dwm/models/mask_layers.py","file_url":"https://github.com/SenseTime-FVG/OpenDWM/blob/HEAD/src/dwm/models/mask_layers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a51979aca82e80a4"}},{"code_sha256_prefix":"074b80faaf135a33","entry":"auto_grad_checkpoint","repo":"SenseTime-FVG/OpenDWM","repo_kind":"official","path":"src/dwm/models/mask_layers.py","file_url":"https://github.com/SenseTime-FVG/OpenDWM/blob/HEAD/src/dwm/models/mask_layers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"074b80faaf135a33"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}