{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/learning-joint-spatial-temporal","title":"Learning Joint Spatial-Temporal Transformations for Video Inpainting","arxiv_id":"2007.10247","date":"2020-07-20","proceeding":"ECCV 2020 8","authors":["Yanhong Zeng","Jianlong Fu","Hongyang Chao"],"abstract":"High-quality video inpainting that completes missing regions in video frames is a promising yet challenging task. State-of-the-art approaches adopt attention models to complete a frame by searching missing contents from reference frames, and further complete whole videos frame by frame. However, these approaches can suffer from inconsistent attention results along spatial and temporal dimensions, which often leads to blurriness and temporal artifacts in videos. In this paper, we propose to learn a joint Spatial-Temporal Transformer Network (STTN) for video inpainting. Specifically, we simultaneously fill missing regions in all input frames by self-attention, and propose to optimize STTN by a spatial-temporal adversarial loss. To show the superiority of the proposed model, we conduct both quantitative and qualitative evaluations by using standard stationary masks and more realistic moving object masks. Demo videos are available at https://github.com/researchmm/STTN.","url_abs":"https://arxiv.org/abs/2007.10247v1","url_pdf":"https://arxiv.org/pdf/2007.10247v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"learning-joint-spatial-temporal","repo_url":"https://github.com/researchmm/STTN","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"learning-joint-spatial-temporal","repo_url":"https://github.com/Feynman1999/MgeEditing","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"unanswered"}}],"tasks":[{"task_slug":"seeing-beyond-the-visible","task_name":"Seeing Beyond the Visible"},{"task_slug":"video-inpainting","task_name":"Video Inpainting"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/seeing-beyond-the-visible-on-kitti360-ex","task":"Seeing Beyond the Visible","dataset":"KITTI360-EX","model":"STTN","rank_in_archive_order":5,"of":7,"metrics":{"Average PSNR":"18.73"},"uses_additional_data":false},{"leaderboard":"/sota/video-inpainting-on-davis","task":"Video Inpainting","dataset":"DAVIS","model":"STTN","rank_in_archive_order":5,"of":11,"metrics":{"Ewarp":"0.1449","PSNR":"30.67","SSIM":"0.9560","VFID":"0.149"},"uses_additional_data":false},{"leaderboard":"/sota/video-inpainting-on-hqvi-240p","task":"Video Inpainting","dataset":"HQVI (240p)","model":"STTN","rank_in_archive_order":7,"of":7,"metrics":{"LPIPS":"0.0528","PSNR":"29.64","SSIM":"0.9339","VFID":"0.2594"},"uses_additional_data":false},{"leaderboard":"/sota/video-inpainting-on-youtube-vos","task":"Video Inpainting","dataset":"YouTube-VOS 2018","model":"STTN","rank_in_archive_order":5,"of":10,"metrics":{"Ewarp":"0.0907","PSNR":"32.34","SSIM":"0.9655","VFID":"0.053"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2007.10247","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2007.10247"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Feynman1999/MgeEditing","reach":{"status":"unanswered"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/researchmm/STTN","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_fixture":1,"unverified":6},"by_repo_kind":{"official":{"samples":7,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"d98f6e80e969ffbd","entry":"random_accelerate","repo":"researchmm/STTN","repo_kind":"official","path":"core/utils.py","file_url":"https://github.com/researchmm/STTN/blob/HEAD/core/utils.py","link_basis":"plan_row","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d98f6e80e969ffbd"}},{"code_sha256_prefix":"30f35d199c193fbe","entry":"create_random_shape_with_random_motion","repo":"researchmm/STTN","repo_kind":"official","path":"core/utils.py","file_url":"https://github.com/researchmm/STTN/blob/HEAD/core/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"30f35d199c193fbe"}},{"code_sha256_prefix":"6377912546960040","entry":"get_random_shape","repo":"researchmm/STTN","repo_kind":"official","path":"core/utils.py","file_url":"https://github.com/researchmm/STTN/blob/HEAD/core/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6377912546960040"}},{"code_sha256_prefix":"460140ea6a070542","entry":"get_ref_index","repo":"researchmm/STTN","repo_kind":"official","path":"core/dataset.py","file_url":"https://github.com/researchmm/STTN/blob/HEAD/core/dataset.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"460140ea6a070542"}},{"code_sha256_prefix":"5db67c73b56c8cd0","entry":"remove_spectral_norm","repo":"researchmm/STTN","repo_kind":"official","path":"core/spectral_norm.py","file_url":"https://github.com/researchmm/STTN/blob/HEAD/core/spectral_norm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5db67c73b56c8cd0"}},{"code_sha256_prefix":"2fbfa7555a2ca4af","entry":"spectral_norm","repo":"researchmm/STTN","repo_kind":"official","path":"core/spectral_norm.py","file_url":"https://github.com/researchmm/STTN/blob/HEAD/core/spectral_norm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2fbfa7555a2ca4af"}},{"code_sha256_prefix":"b674053606e02d7e","entry":"use_spectral_norm","repo":"researchmm/STTN","repo_kind":"official","path":"core/spectral_norm.py","file_url":"https://github.com/researchmm/STTN/blob/HEAD/core/spectral_norm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b674053606e02d7e"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}