{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/flatten-optical-flow-guided-attention-for","title":"FLATTEN: optical FLow-guided ATTENtion for consistent text-to-video editing","arxiv_id":"2310.05922","date":"2023-10-09","proceeding":null,"authors":["Yuren Cong","Mengmeng Xu","Christian Simon","Shoufa Chen","Jiawei Ren","Yanping Xie","Juan-Manuel Perez-Rua","Bodo Rosenhahn","Tao Xiang","Sen He"],"abstract":"Text-to-video editing aims to edit the visual appearance of a source video conditional on textual prompts. A major challenge in this task is to ensure that all frames in the edited video are visually consistent. Most recent works apply advanced text-to-image diffusion models to this task by inflating 2D spatial attention in the U-Net into spatio-temporal attention. Although temporal context can be added through spatio-temporal attention, it may introduce some irrelevant information for each patch and therefore cause inconsistency in the edited video. In this paper, for the first time, we introduce optical flow into the attention module in the diffusion model's U-Net to address the inconsistency issue for text-to-video editing. Our method, FLATTEN, enforces the patches on the same flow path across different frames to attend to each other in the attention module, thus improving the visual consistency in the edited videos. Additionally, our method is training-free and can be seamlessly integrated into any diffusion-based text-to-video editing methods and improve their visual consistency. Experiment results on existing text-to-video editing benchmarks show that our proposed method achieves the new state-of-the-art performance. In particular, our method excels in maintaining the visual consistency in the edited videos.","url_abs":"https://arxiv.org/abs/2310.05922v3","url_pdf":"https://arxiv.org/pdf/2310.05922v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"flatten-optical-flow-guided-attention-for","repo_url":"https://github.com/yrcong/flatten","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"optical-flow-estimation","task_name":"Optical Flow Estimation"},{"task_slug":"text-to-video-editing","task_name":"Text-to-Video Editing"},{"task_slug":"video-editing","task_name":"Video Editing"}],"methods":[{"method_slug":"concatenated-skip-connection","method_name":"Concatenated Skip Connection"},{"method_slug":"convolution","method_name":"Convolution"},{"method_slug":"diffusion","method_name":"Diffusion"},{"method_slug":"max-pooling","method_name":"Max Pooling"},{"method_slug":"relu","method_name":"ReLU"},{"method_slug":"u-net","method_name":"U-Net"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2310.05922","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.05922"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yrcong/flatten","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":1,"unverified":3},"by_repo_kind":{"listed":{"samples":4,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"fb032d6fa7e6196d","entry":"next_step","repo":"yrcong/flatten","repo_kind":"listed","path":"models/util.py","file_url":"https://github.com/yrcong/flatten/blob/HEAD/models/util.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fb032d6fa7e6196d"}},{"code_sha256_prefix":"cd04c520728fce0f","entry":"get_down_block","repo":"yrcong/flatten","repo_kind":"listed","path":"models/unet_blocks.py","file_url":"https://github.com/yrcong/flatten/blob/HEAD/models/unet_blocks.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cd04c520728fce0f"}},{"code_sha256_prefix":"418c4f92e19dccad","entry":"get_up_block","repo":"yrcong/flatten","repo_kind":"listed","path":"models/unet_blocks.py","file_url":"https://github.com/yrcong/flatten/blob/HEAD/models/unet_blocks.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"418c4f92e19dccad"}},{"code_sha256_prefix":"a0a4d94af120c56f","entry":"init_prompt","repo":"yrcong/flatten","repo_kind":"listed","path":"models/util.py","file_url":"https://github.com/yrcong/flatten/blob/HEAD/models/util.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a0a4d94af120c56f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}