{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/rstt-real-time-spatial-temporal-transformer","title":"RSTT: Real-time Spatial Temporal Transformer for Space-Time Video Super-Resolution","arxiv_id":"2203.14186","date":"2022-03-27","proceeding":"CVPR 2022 1","authors":["Zhicheng Geng","Luming Liang","Tianyu Ding","Ilya Zharkov"],"abstract":"Space-time video super-resolution (STVSR) is the task of interpolating videos with both Low Frame Rate (LFR) and Low Resolution (LR) to produce High-Frame-Rate (HFR) and also High-Resolution (HR) counterparts. The existing methods based on Convolutional Neural Network~(CNN) succeed in achieving visually satisfied results while suffer from slow inference speed due to their heavy architectures. We propose to resolve this issue by using a spatial-temporal transformer that naturally incorporates the spatial and temporal super resolution modules into a single model. Unlike CNN-based methods, we do not explicitly use separated building blocks for temporal interpolations and spatial super-resolutions; instead, we only use a single end-to-end transformer architecture. Specifically, a reusable dictionary is built by encoders based on the input LFR and LR frames, which is then utilized in the decoder part to synthesize the HFR and HR frames. Compared with the state-of-the-art TMNet \\cite{xu2021temporal}, our network is $60\\%$ smaller (4.5M vs 12.3M parameters) and $80\\%$ faster (26.2fps vs 14.3fps on $720\\times576$ frames) without sacrificing much performance. The source code is available at https://github.com/llmpass/RSTT.","url_abs":"https://arxiv.org/abs/2203.14186v1","url_pdf":"https://arxiv.org/pdf/2203.14186v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"rstt-real-time-spatial-temporal-transformer","repo_url":"https://github.com/llmpass/RSTT","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"decoder","task_name":"Decoder"},{"task_slug":"space-time-video-super-resolution","task_name":"Space-time Video Super-resolution"},{"task_slug":"super-resolution","task_name":"Super-Resolution"},{"task_slug":"video-frame-interpolation","task_name":"Video Frame Interpolation"},{"task_slug":"video-super-resolution","task_name":"Video Super-Resolution"}],"methods":[{"method_slug":"attention","method_name":"Attention"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"speed","method_name":"SPEED"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"stochastic-depth","method_name":"Stochastic Depth"},{"method_slug":"swin-transformer","method_name":"Swin Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/space-time-video-super-resolution-on-vimeo90k","task":"Space-time Video Super-resolution","dataset":"Vimeo90K-Fast","model":"RSTT-L","rank_in_archive_order":2,"of":4,"metrics":{"PSNR":"36.80","SSIM":"0.9403"},"uses_additional_data":false},{"leaderboard":"/sota/space-time-video-super-resolution-on-vimeo90k","task":"Space-time Video Super-resolution","dataset":"Vimeo90K-Fast","model":"RSTT-M","rank_in_archive_order":3,"of":4,"metrics":{"PSNR":"36.78","SSIM":"0.9401"},"uses_additional_data":false},{"leaderboard":"/sota/space-time-video-super-resolution-on-vimeo90k","task":"Space-time Video Super-resolution","dataset":"Vimeo90K-Fast","model":"RSTT-S","rank_in_archive_order":4,"of":4,"metrics":{"PSNR":"36.58","SSIM":"0.9381"},"uses_additional_data":false},{"leaderboard":"/sota/space-time-video-super-resolution-on-vimeo90k-1","task":"Space-time Video Super-resolution","dataset":"Vimeo90K-Medium","model":"RSTT-L","rank_in_archive_order":2,"of":4,"metrics":{"PSNR":"35.66","SSIM":"0.9381"},"uses_additional_data":false},{"leaderboard":"/sota/space-time-video-super-resolution-on-vimeo90k-1","task":"Space-time Video Super-resolution","dataset":"Vimeo90K-Medium","model":"RSTT-M","rank_in_archive_order":3,"of":4,"metrics":{"PSNR":"35.62","SSIM":"0.9377"},"uses_additional_data":false},{"leaderboard":"/sota/space-time-video-super-resolution-on-vimeo90k-1","task":"Space-time Video Super-resolution","dataset":"Vimeo90K-Medium","model":"RSTT-S","rank_in_archive_order":4,"of":4,"metrics":{"PSNR":"35.43","SSIM":"0.9358"},"uses_additional_data":false},{"leaderboard":"/sota/video-frame-interpolation-on-vid4-4x","task":"Video Frame Interpolation","dataset":"Vid4 - 4x upscaling","model":"RSTT-L","rank_in_archive_order":2,"of":5,"metrics":{"PSNR":"26.43","Parameters":"7670000","SSIM":"0.7994"},"uses_additional_data":false},{"leaderboard":"/sota/video-frame-interpolation-on-vid4-4x","task":"Video Frame Interpolation","dataset":"Vid4 - 4x upscaling","model":"RSTT-M","rank_in_archive_order":3,"of":5,"metrics":{"PSNR":"26.37","Parameters":"6080000","SSIM":"0.7978"},"uses_additional_data":false},{"leaderboard":"/sota/video-frame-interpolation-on-vid4-4x","task":"Video Frame Interpolation","dataset":"Vid4 - 4x upscaling","model":"RSTT-S","rank_in_archive_order":5,"of":5,"metrics":{"PSNR":"26.29","Parameters":"4490000","SSIM":"0.7941"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2203.14186","atlas_url":"https://app.syntology.ai/?focus=2203.14186","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.14186"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/llmpass/RSTT","reach":null}],"summary":{"ran":5,"ran_fixture":2,"ran_draft_wrong":1,"unverified":6},"by_repo_kind":{"official":{"samples":14,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":14,"samples":[{"code_sha256_prefix":"190b1325503c4ab8","entry":"Downsample","repo":"llmpass/RSTT","repo_kind":"official","path":"models/RSTT.py","file_url":"https://github.com/llmpass/RSTT/blob/HEAD/models/RSTT.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"190b1325503c4ab8"}},{"code_sha256_prefix":"3aee7fafe363d322","entry":"InputProj","repo":"llmpass/RSTT","repo_kind":"official","path":"models/RSTT.py","file_url":"https://github.com/llmpass/RSTT/blob/HEAD/models/RSTT.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3aee7fafe363d322"}},{"code_sha256_prefix":"b80f785c8fa3697e","entry":"ResidualBlock_noBN","repo":"llmpass/RSTT","repo_kind":"official","path":"models/RSTT.py","file_url":"https://github.com/llmpass/RSTT/blob/HEAD/models/RSTT.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b80f785c8fa3697e"}},{"code_sha256_prefix":"9d6b9fecff821851","entry":"Upsample","repo":"llmpass/RSTT","repo_kind":"official","path":"models/RSTT.py","file_url":"https://github.com/llmpass/RSTT/blob/HEAD/models/RSTT.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9d6b9fecff821851"}},{"code_sha256_prefix":"6f69c55b59c9744c","entry":"WindowAttention3D","repo":"llmpass/RSTT","repo_kind":"official","path":"models/RSTT.py","file_url":"https://github.com/llmpass/RSTT/blob/HEAD/models/RSTT.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6f69c55b59c9744c"}},{"code_sha256_prefix":"d306f10d62b6ae0c","entry":"get_window_size","repo":"llmpass/RSTT","repo_kind":"official","path":"models/RSTT.py","file_url":"https://github.com/llmpass/RSTT/blob/HEAD/models/RSTT.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d306f10d62b6ae0c"}},{"code_sha256_prefix":"dbf81aece2a63747","entry":"window_partition","repo":"llmpass/RSTT","repo_kind":"official","path":"models/RSTT.py","file_url":"https://github.com/llmpass/RSTT/blob/HEAD/models/RSTT.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"dbf81aece2a63747"}},{"code_sha256_prefix":"15e141e09e3e5d8d","entry":"window_reverse","repo":"llmpass/RSTT","repo_kind":"official","path":"models/RSTT.py","file_url":"https://github.com/llmpass/RSTT/blob/HEAD/models/RSTT.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"15e141e09e3e5d8d"}},{"code_sha256_prefix":"cb3cd3a0e1663f15","entry":"DecoderLayer","repo":"llmpass/RSTT","repo_kind":"official","path":"models/RSTT.py","file_url":"https://github.com/llmpass/RSTT/blob/HEAD/models/RSTT.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"cb3cd3a0e1663f15"}},{"code_sha256_prefix":"8e283aa9300b35c3","entry":"EncoderLayer","repo":"llmpass/RSTT","repo_kind":"official","path":"models/RSTT.py","file_url":"https://github.com/llmpass/RSTT/blob/HEAD/models/RSTT.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8e283aa9300b35c3"}},{"code_sha256_prefix":"6c382b558756a30b","entry":"RSTT","repo":"llmpass/RSTT","repo_kind":"official","path":"models/RSTT.py","file_url":"https://github.com/llmpass/RSTT/blob/HEAD/models/RSTT.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6c382b558756a30b"}},{"code_sha256_prefix":"5b1fddb874b78287","entry":"VSTSRDecoderTransformerBlock","repo":"llmpass/RSTT","repo_kind":"official","path":"models/RSTT.py","file_url":"https://github.com/llmpass/RSTT/blob/HEAD/models/RSTT.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5b1fddb874b78287"}},{"code_sha256_prefix":"2206f8156a0c0bc6","entry":"VSTSREncoderTransformerBlock","repo":"llmpass/RSTT","repo_kind":"official","path":"models/RSTT.py","file_url":"https://github.com/llmpass/RSTT/blob/HEAD/models/RSTT.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2206f8156a0c0bc6"}},{"code_sha256_prefix":"e217d1b65eaca404","entry":"initialize_weights","repo":"llmpass/RSTT","repo_kind":"official","path":"models/RSTT.py","file_url":"https://github.com/llmpass/RSTT/blob/HEAD/models/RSTT.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e217d1b65eaca404"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}