{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/looking-backward-streaming-video-to-video","title":"Looking Backward: Streaming Video-to-Video Translation with Feature Banks","arxiv_id":"2405.15757","date":"2024-05-24","proceeding":null,"authors":["Feng Liang","Akio Kodaira","Chenfeng Xu","Masayoshi Tomizuka","Kurt Keutzer","Diana Marculescu"],"abstract":"This paper introduces StreamV2V, a diffusion model that achieves real-time streaming video-to-video (V2V) translation with user prompts. Unlike prior V2V methods using batches to process limited frames, we opt to process frames in a streaming fashion, to support unlimited frames. At the heart of StreamV2V lies a backward-looking principle that relates the present to the past. This is realized by maintaining a feature bank, which archives information from past frames. For incoming frames, StreamV2V extends self-attention to include banked keys and values and directly fuses similar past features into the output. The feature bank is continually updated by merging stored and new features, making it compact but informative. StreamV2V stands out for its adaptability and efficiency, seamlessly integrating with image diffusion models without fine-tuning. It can run 20 FPS on one A100 GPU, being 15x, 46x, 108x, and 158x faster than FlowVid, CoDeF, Rerender, and TokenFlow, respectively. Quantitative metrics and user studies confirm StreamV2V's exceptional ability to maintain temporal consistency.","url_abs":"https://arxiv.org/abs/2405.15757v3","url_pdf":"https://arxiv.org/pdf/2405.15757v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"looking-backward-streaming-video-to-video","repo_url":"https://github.com/Jeff-LiangF/streamv2v","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}}],"tasks":[{"task_slug":null,"task_name":"GPU"},{"task_slug":"translation","task_name":"Translation"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"},{"method_slug":"opt","method_name":"OPT"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2405.15757","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.15757"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Jeff-LiangF/streamv2v","reach":{"status":"ok","spdx":"NOASSERTION"}}],"summary":{"ran":10,"unverified":2},"by_repo_kind":{"official":{"samples":12,"ran":10,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":12,"samples":[{"code_sha256_prefix":"d1f7607f85bbbcf6","entry":"create_onnx_path","repo":"Jeff-LiangF/streamv2v","repo_kind":"official","path":"src/streamv2v/acceleration/tensorrt/builder.py","file_url":"https://github.com/Jeff-LiangF/streamv2v/blob/HEAD/src/streamv2v/acceleration/tensorrt/builder.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"d1f7607f85bbbcf6"}},{"code_sha256_prefix":"16ce98a3e3130938","entry":"decode_images","repo":"Jeff-LiangF/streamv2v","repo_kind":"official","path":"src/streamv2v/acceleration/tensorrt/utilities.py","file_url":"https://github.com/Jeff-LiangF/streamv2v/blob/HEAD/src/streamv2v/acceleration/tensorrt/utilities.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"16ce98a3e3130938"}},{"code_sha256_prefix":"5dd5ccc011afb9c3","entry":"denormalize","repo":"Jeff-LiangF/streamv2v","repo_kind":"official","path":"src/streamv2v/image_utils.py","file_url":"https://github.com/Jeff-LiangF/streamv2v/blob/HEAD/src/streamv2v/image_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"5dd5ccc011afb9c3"}},{"code_sha256_prefix":"b2ca37137e7e5c8b","entry":"get_nn_feats","repo":"Jeff-LiangF/streamv2v","repo_kind":"official","path":"src/streamv2v/models/utils.py","file_url":"https://github.com/Jeff-LiangF/streamv2v/blob/HEAD/src/streamv2v/models/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"b2ca37137e7e5c8b"}},{"code_sha256_prefix":"35d471d9e81ee42a","entry":"is_installed","repo":"Jeff-LiangF/streamv2v","repo_kind":"official","path":"src/streamv2v/pip_utils.py","file_url":"https://github.com/Jeff-LiangF/streamv2v/blob/HEAD/src/streamv2v/pip_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"35d471d9e81ee42a"}},{"code_sha256_prefix":"b188597984f52631","entry":"numpy_to_pil","repo":"Jeff-LiangF/streamv2v","repo_kind":"official","path":"src/streamv2v/image_utils.py","file_url":"https://github.com/Jeff-LiangF/streamv2v/blob/HEAD/src/streamv2v/image_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"b188597984f52631"}},{"code_sha256_prefix":"7f011624871aea53","entry":"preprocess_image","repo":"Jeff-LiangF/streamv2v","repo_kind":"official","path":"src/streamv2v/acceleration/tensorrt/utilities.py","file_url":"https://github.com/Jeff-LiangF/streamv2v/blob/HEAD/src/streamv2v/acceleration/tensorrt/utilities.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"7f011624871aea53"}},{"code_sha256_prefix":"e0ca6f492a8f2cbf","entry":"pt_to_numpy","repo":"Jeff-LiangF/streamv2v","repo_kind":"official","path":"src/streamv2v/image_utils.py","file_url":"https://github.com/Jeff-LiangF/streamv2v/blob/HEAD/src/streamv2v/image_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"e0ca6f492a8f2cbf"}},{"code_sha256_prefix":"5537486cf51ea735","entry":"random_bipartite_soft_matching","repo":"Jeff-LiangF/streamv2v","repo_kind":"official","path":"src/streamv2v/models/utils.py","file_url":"https://github.com/Jeff-LiangF/streamv2v/blob/HEAD/src/streamv2v/models/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"5537486cf51ea735"}},{"code_sha256_prefix":"635ea667a5c67443","entry":"version","repo":"Jeff-LiangF/streamv2v","repo_kind":"official","path":"src/streamv2v/pip_utils.py","file_url":"https://github.com/Jeff-LiangF/streamv2v/blob/HEAD/src/streamv2v/pip_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"635ea667a5c67443"}},{"code_sha256_prefix":"ebafb3d27531a089","entry":"get_nn_latent","repo":"Jeff-LiangF/streamv2v","repo_kind":"official","path":"src/streamv2v/models/utils.py","file_url":"https://github.com/Jeff-LiangF/streamv2v/blob/HEAD/src/streamv2v/models/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"ebafb3d27531a089"}},{"code_sha256_prefix":"ef16b546127f1484","entry":"run_python","repo":"Jeff-LiangF/streamv2v","repo_kind":"official","path":"src/streamv2v/pip_utils.py","file_url":"https://github.com/Jeff-LiangF/streamv2v/blob/HEAD/src/streamv2v/pip_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"ef16b546127f1484"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}