{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/videotetris-towards-compositional-text-to","title":"VideoTetris: Towards Compositional Text-to-Video Generation","arxiv_id":"2406.04277","date":"2024-06-06","proceeding":null,"authors":["Ye Tian","Ling Yang","Haotian Yang","Yuan Gao","Yufan Deng","Jingmin Chen","Xintao Wang","Zhaochen Yu","Xin Tao","Pengfei Wan","Di Zhang","Bin Cui"],"abstract":"Diffusion models have demonstrated great success in text-to-video (T2V) generation. However, existing methods may face challenges when handling complex (long) video generation scenarios that involve multiple objects or dynamic changes in object numbers. To address these limitations, we propose VideoTetris, a novel framework that enables compositional T2V generation. Specifically, we propose spatio-temporal compositional diffusion to precisely follow complex textual semantics by manipulating and composing the attention maps of denoising networks spatially and temporally. Moreover, we propose an enhanced video data preprocessing to enhance the training data regarding motion dynamics and prompt understanding, equipped with a new reference frame attention mechanism to improve the consistency of auto-regressive video generation. Extensive experiments demonstrate that our VideoTetris achieves impressive qualitative and quantitative results in compositional T2V generation. Code is available at: https://github.com/YangLing0818/VideoTetris","url_abs":"https://arxiv.org/abs/2406.04277v2","url_pdf":"https://arxiv.org/pdf/2406.04277v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"videotetris-towards-compositional-text-to","repo_url":"https://github.com/yangling0818/videotetris","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"denoising","task_name":"Denoising"},{"task_slug":"text-to-video-generation","task_name":"Text-to-Video Generation"},{"task_slug":"video-generation","task_name":"Video Generation"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2406.04277","atlas_url":"https://app.syntology.ai/?focus=2406.04277","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.04277"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yangling0818/videotetris","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":5,"ran_draft_wrong":1,"unverified":3},"by_repo_kind":{"official":{"samples":9,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"ea5ebc1fccaecd97","entry":"ad_short_gen","repo":"yangling0818/videotetris","repo_kind":"official","path":"long/t2v_enhanced/model_func.py","file_url":"https://github.com/yangling0818/videotetris/blob/HEAD/long/t2v_enhanced/model_func.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ea5ebc1fccaecd97"}},{"code_sha256_prefix":"095f09e68e8de65c","entry":"add_margin","repo":"yangling0818/videotetris","repo_kind":"official","path":"long/t2v_enhanced/inference_utils.py","file_url":"https://github.com/yangling0818/videotetris/blob/HEAD/long/t2v_enhanced/inference_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"095f09e68e8de65c"}},{"code_sha256_prefix":"2fefafd463091638","entry":"legacy_transformation","repo":"yangling0818/videotetris","repo_kind":"official","path":"long/t2v_enhanced/inference_utils.py","file_url":"https://github.com/yangling0818/videotetris/blob/HEAD/long/t2v_enhanced/inference_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2fefafd463091638"}},{"code_sha256_prefix":"9d04b073bb0e55c8","entry":"remove_value","repo":"yangling0818/videotetris","repo_kind":"official","path":"long/t2v_enhanced/inference_utils.py","file_url":"https://github.com/yangling0818/videotetris/blob/HEAD/long/t2v_enhanced/inference_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9d04b073bb0e55c8"}},{"code_sha256_prefix":"9fd4368f1cc738db","entry":"sdxl_image_gen","repo":"yangling0818/videotetris","repo_kind":"official","path":"long/t2v_enhanced/model_func.py","file_url":"https://github.com/yangling0818/videotetris/blob/HEAD/long/t2v_enhanced/model_func.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9fd4368f1cc738db"}},{"code_sha256_prefix":"52ed01399226d3df","entry":"zero_module","repo":"YangLing0818/VideoTetris","repo_kind":"official","path":"long/t2v_enhanced/model/diffusers_conditional/models/controlnet/controlnet.py","file_url":"https://github.com/YangLing0818/VideoTetris/blob/HEAD/long/t2v_enhanced/model/diffusers_conditional/models/controlnet/controlnet.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"52ed01399226d3df"}},{"code_sha256_prefix":"e4b3bfff9427f7f7","entry":"auto_str","repo":"yangling0818/videotetris","repo_kind":"official","path":"long/t2v_enhanced/model/pl_module_params_controlnet.py","file_url":"https://github.com/yangling0818/videotetris/blob/HEAD/long/t2v_enhanced/model/pl_module_params_controlnet.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e4b3bfff9427f7f7"}},{"code_sha256_prefix":"3e997c19843d1786","entry":"gaussian_smoothing_kernel","repo":"yangling0818/videotetris","repo_kind":"official","path":"long/t2v_enhanced/model/video_noise_generator.py","file_url":"https://github.com/yangling0818/videotetris/blob/HEAD/long/t2v_enhanced/model/video_noise_generator.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3e997c19843d1786"}},{"code_sha256_prefix":"a2613130a496d371","entry":"ms_short_gen","repo":"yangling0818/videotetris","repo_kind":"official","path":"long/t2v_enhanced/model_func.py","file_url":"https://github.com/yangling0818/videotetris/blob/HEAD/long/t2v_enhanced/model_func.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a2613130a496d371"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}