{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/diffusion-models-for-video-prediction-and","title":"Diffusion Models for Video Prediction and Infilling","arxiv_id":"2206.07696","date":"2022-06-15","proceeding":null,"authors":["Tobias Höppe","Arash Mehrjou","Stefan Bauer","Didrik Nielsen","Andrea Dittadi"],"abstract":"Predicting and anticipating future outcomes or reasoning about missing information in a sequence are critical skills for agents to be able to make intelligent decisions. This requires strong, temporally coherent generative capabilities. Diffusion models have shown remarkable success in several generative tasks, but have not been extensively explored in the video domain. We present Random-Mask Video Diffusion (RaMViD), which extends image diffusion models to videos using 3D convolutions, and introduces a new conditioning technique during training. By varying the mask we condition on, the model is able to perform video prediction, infilling, and upsampling. Due to our simple conditioning scheme, we can utilize the same architecture as used for unconditional training, which allows us to train the model in a conditional and unconditional fashion at the same time. We evaluate RaMViD on two benchmark datasets for video prediction, on which we achieve state-of-the-art results, and one for video generation. High-resolution videos are provided at https://sites.google.com/view/video-diffusion-prediction.","url_abs":"https://arxiv.org/abs/2206.07696v3","url_pdf":"https://arxiv.org/pdf/2206.07696v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"diffusion-models-for-video-prediction-and","repo_url":"https://github.com/Tobi-r9/RaMViD","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"prediction","task_name":"Prediction"},{"task_slug":"video-generation","task_name":"Video Generation"},{"task_slug":"video-prediction","task_name":"Video Prediction"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/video-generation-on-bair-robot-pushing","task":"Video Generation","dataset":"BAIR Robot Pushing","model":"RaMViD","rank_in_archive_order":2,"of":31,"metrics":{"Cond":"1","FVD score":"84.20","Pred":"15","Train":"20"},"uses_additional_data":false},{"leaderboard":"/sota/video-prediction-on-kinetics-600-12-frames","task":"Video Prediction","dataset":"Kinetics-600 12 frames, 64x64","model":"RaMViD","rank_in_archive_order":8,"of":16,"metrics":{"Cond":"5","FVD":"16.46","Pred":"11"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2206.07696","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2206.07696"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Tobi-r9/RaMViD","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":3,"ran_honours":4,"ran":1,"ran_fixture":1,"unverified":10},"by_repo_kind":{"official":{"samples":19,"ran":9,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"cfd76fd0d89574a4","entry":"approx_standard_normal_cdf","repo":"Tobi-r9/RaMViD","repo_kind":"official","path":"diffusion_openai/losses.py","file_url":"https://github.com/Tobi-r9/RaMViD/blob/HEAD/diffusion_openai/losses.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cfd76fd0d89574a4"}},{"code_sha256_prefix":"2ab2316ac6fdd869","entry":"betas_for_alpha_bar","repo":"Tobi-r9/RaMViD","repo_kind":"official","path":"diffusion_openai/gaussian_diffusion.py","file_url":"https://github.com/Tobi-r9/RaMViD/blob/HEAD/diffusion_openai/gaussian_diffusion.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2ab2316ac6fdd869"}},{"code_sha256_prefix":"cd33283d615fb3d7","entry":"discretized_gaussian_log_likelihood","repo":"Tobi-r9/RaMViD","repo_kind":"official","path":"diffusion_openai/losses.py","file_url":"https://github.com/Tobi-r9/RaMViD/blob/HEAD/diffusion_openai/losses.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cd33283d615fb3d7"}},{"code_sha256_prefix":"a086d6286a40b889","entry":"get_named_beta_schedule","repo":"Tobi-r9/RaMViD","repo_kind":"official","path":"diffusion_openai/gaussian_diffusion.py","file_url":"https://github.com/Tobi-r9/RaMViD/blob/HEAD/diffusion_openai/gaussian_diffusion.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a086d6286a40b889"}},{"code_sha256_prefix":"bcd8b4acab199405","entry":"make_output_format","repo":"Tobi-r9/RaMViD","repo_kind":"official","path":"diffusion_openai/logger.py","file_url":"https://github.com/Tobi-r9/RaMViD/blob/HEAD/diffusion_openai/logger.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bcd8b4acab199405"}},{"code_sha256_prefix":"cf2798b666b231ca","entry":"normal_kl","repo":"Tobi-r9/RaMViD","repo_kind":"official","path":"diffusion_openai/losses.py","file_url":"https://github.com/Tobi-r9/RaMViD/blob/HEAD/diffusion_openai/losses.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cf2798b666b231ca"}},{"code_sha256_prefix":"76313eaf1a6e8f2a","entry":"parse_resume_step_from_filename","repo":"Tobi-r9/RaMViD","repo_kind":"official","path":"diffusion_openai/train_util.py","file_url":"https://github.com/Tobi-r9/RaMViD/blob/HEAD/diffusion_openai/train_util.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"76313eaf1a6e8f2a"}},{"code_sha256_prefix":"9b6e4e070473e76f","entry":"space_timesteps","repo":"Tobi-r9/RaMViD","repo_kind":"official","path":"diffusion_openai/respace.py","file_url":"https://github.com/Tobi-r9/RaMViD/blob/HEAD/diffusion_openai/respace.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9b6e4e070473e76f"}},{"code_sha256_prefix":"129b804760b3115f","entry":"zero_module","repo":"Tobi-r9/RaMViD","repo_kind":"official","path":"diffusion_openai/nn.py","file_url":"https://github.com/Tobi-r9/RaMViD/blob/HEAD/diffusion_openai/nn.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"129b804760b3115f"}},{"code_sha256_prefix":"ecd0fc28815b65ae","entry":"avg_pool_nd","repo":"Tobi-r9/RaMViD","repo_kind":"official","path":"diffusion_openai/nn.py","file_url":"https://github.com/Tobi-r9/RaMViD/blob/HEAD/diffusion_openai/nn.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ecd0fc28815b65ae"}},{"code_sha256_prefix":"fe4eb545bbb728e0","entry":"conv_nd","repo":"Tobi-r9/RaMViD","repo_kind":"official","path":"diffusion_openai/nn.py","file_url":"https://github.com/Tobi-r9/RaMViD/blob/HEAD/diffusion_openai/nn.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fe4eb545bbb728e0"}},{"code_sha256_prefix":"3ff9a4f729f580ca","entry":"create_model","repo":"Tobi-r9/RaMViD","repo_kind":"official","path":"diffusion_openai/script_util.py","file_url":"https://github.com/Tobi-r9/RaMViD/blob/HEAD/diffusion_openai/script_util.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3ff9a4f729f580ca"}},{"code_sha256_prefix":"62c3113f19ddedd4","entry":"create_model_and_diffusion","repo":"Tobi-r9/RaMViD","repo_kind":"official","path":"diffusion_openai/script_util.py","file_url":"https://github.com/Tobi-r9/RaMViD/blob/HEAD/diffusion_openai/script_util.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"62c3113f19ddedd4"}},{"code_sha256_prefix":"f882d2f9d0d962e2","entry":"create_named_schedule_sampler","repo":"Tobi-r9/RaMViD","repo_kind":"official","path":"diffusion_openai/resample.py","file_url":"https://github.com/Tobi-r9/RaMViD/blob/HEAD/diffusion_openai/resample.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f882d2f9d0d962e2"}},{"code_sha256_prefix":"a863803cdd5f3ce6","entry":"make_master_params","repo":"Tobi-r9/RaMViD","repo_kind":"official","path":"diffusion_openai/fp16_util.py","file_url":"https://github.com/Tobi-r9/RaMViD/blob/HEAD/diffusion_openai/fp16_util.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a863803cdd5f3ce6"}},{"code_sha256_prefix":"e515a67f7f32e76d","entry":"mpi_weighted_mean","repo":"Tobi-r9/RaMViD","repo_kind":"official","path":"diffusion_openai/logger.py","file_url":"https://github.com/Tobi-r9/RaMViD/blob/HEAD/diffusion_openai/logger.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e515a67f7f32e76d"}},{"code_sha256_prefix":"0c6607473a4c4c55","entry":"profile","repo":"Tobi-r9/RaMViD","repo_kind":"official","path":"diffusion_openai/logger.py","file_url":"https://github.com/Tobi-r9/RaMViD/blob/HEAD/diffusion_openai/logger.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0c6607473a4c4c55"}},{"code_sha256_prefix":"81d6677cc60fb151","entry":"sr_create_model_and_diffusion","repo":"Tobi-r9/RaMViD","repo_kind":"official","path":"diffusion_openai/script_util.py","file_url":"https://github.com/Tobi-r9/RaMViD/blob/HEAD/diffusion_openai/script_util.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"81d6677cc60fb151"}},{"code_sha256_prefix":"30e43bcf12d042b0","entry":"unflatten_master_params","repo":"Tobi-r9/RaMViD","repo_kind":"official","path":"diffusion_openai/fp16_util.py","file_url":"https://github.com/Tobi-r9/RaMViD/blob/HEAD/diffusion_openai/fp16_util.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"30e43bcf12d042b0"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}