{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/vrt-a-video-restoration-transformer","title":"VRT: A Video Restoration Transformer","arxiv_id":"2201.12288","date":"2022-01-28","proceeding":null,"authors":["Jingyun Liang","JieZhang Cao","Yuchen Fan","Kai Zhang","Rakesh Ranjan","Yawei Li","Radu Timofte","Luc van Gool"],"abstract":"Video restoration (e.g., video super-resolution) aims to restore high-quality frames from low-quality frames. Different from single image restoration, video restoration generally requires to utilize temporal information from multiple adjacent but usually misaligned video frames. Existing deep methods generally tackle with this by exploiting a sliding window strategy or a recurrent architecture, which either is restricted by frame-by-frame restoration or lacks long-range modelling ability. In this paper, we propose a Video Restoration Transformer (VRT) with parallel frame prediction and long-range temporal dependency modelling abilities. More specifically, VRT is composed of multiple scales, each of which consists of two kinds of modules: temporal mutual self attention (TMSA) and parallel warping. TMSA divides the video into small clips, on which mutual attention is applied for joint motion estimation, feature alignment and feature fusion, while self attention is used for feature extraction. To enable cross-clip interactions, the video sequence is shifted for every other layer. Besides, parallel warping is used to further fuse information from neighboring frames by parallel feature warping. Experimental results on five tasks, including video super-resolution, video deblurring, video denoising, video frame interpolation and space-time video super-resolution, demonstrate that VRT outperforms the state-of-the-art methods by large margins ($\\textbf{up to 2.16dB}$) on fourteen benchmark datasets.","url_abs":"https://arxiv.org/abs/2201.12288v2","url_pdf":"https://arxiv.org/pdf/2201.12288v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"vrt-a-video-restoration-transformer","repo_url":"https://github.com/jingyunliang/vrt","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"deblurring","task_name":"Deblurring"},{"task_slug":"denoising","task_name":"Denoising"},{"task_slug":"image-restoration","task_name":"Image Restoration"},{"task_slug":"motion-estimation","task_name":"Motion Estimation"},{"task_slug":"space-time-video-super-resolution","task_name":"Space-time Video Super-resolution"},{"task_slug":"super-resolution","task_name":"Super-Resolution"},{"task_slug":"video-deblurring","task_name":"Video Deblurring"},{"task_slug":"video-denoising","task_name":"Video Denoising"},{"task_slug":"video-frame-interpolation","task_name":"Video Frame Interpolation"},{"task_slug":"video-restoration","task_name":"Video Restoration"},{"task_slug":"video-super-resolution","task_name":"Video Super-Resolution"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/deblurring-on-based-1","task":"Deblurring","dataset":"BASED","model":"VRT (GoPro)","rank_in_archive_order":1,"of":4,"metrics":{"PSNR":"31.42945","VMAF":"66.72253"},"uses_additional_data":false},{"leaderboard":"/sota/deblurring-on-based-1","task":"Deblurring","dataset":"BASED","model":"VRT (REDS)","rank_in_archive_order":3,"of":4,"metrics":{"ERQAv2.0":"0.74874","LPIPS":"0.08248","PSNR":"30.97878","SSIM":"0.94503","VMAF":"66.81782"},"uses_additional_data":false},{"leaderboard":"/sota/deblurring-on-dvd-1","task":"Deblurring","dataset":"DVD","model":"VRT","rank_in_archive_order":4,"of":7,"metrics":{"PSNR":"34.27"},"uses_additional_data":false},{"leaderboard":"/sota/deblurring-on-gopro","task":"Deblurring","dataset":"GoPro","model":"VRT","rank_in_archive_order":6,"of":56,"metrics":{"PSNR":"34.81","SSIM":"0.9724"},"uses_additional_data":true},{"leaderboard":"/sota/deblurring-on-based","task":"Deblurring","dataset":"MSU BASED","model":"VRT (GoPro)","rank_in_archive_order":2,"of":11,"metrics":{"ERQAv2.0":"0.74874","LPIPS":"0.08165","PSNR":"31.42945","SSIM":"0.94503","Subjective":"2.3854","VMAF":"66.72253"},"uses_additional_data":false},{"leaderboard":"/sota/deblurring-on-based","task":"Deblurring","dataset":"MSU BASED","model":"VRT (REDS)","rank_in_archive_order":3,"of":11,"metrics":{"ERQAv2.0":"0.75056","LPIPS":"0.08248","PSNR":"30.97878","SSIM":"0.94601","Subjective":"1.5660","VMAF":"66.81782"},"uses_additional_data":false},{"leaderboard":"/sota/deblurring-on-reds","task":"Deblurring","dataset":"REDS","model":"VRT","rank_in_archive_order":1,"of":3,"metrics":{"Average PSNR":"36.79"},"uses_additional_data":false},{"leaderboard":"/sota/space-time-video-super-resolution-on-vimeo90k","task":"Space-time Video Super-resolution","dataset":"Vimeo90K-Fast","model":"VRT","rank_in_archive_order":1,"of":4,"metrics":{"PSNR":"36.98","SSIM":"0.9439"},"uses_additional_data":false},{"leaderboard":"/sota/space-time-video-super-resolution-on-vimeo90k-1","task":"Space-time Video Super-resolution","dataset":"Vimeo90K-Medium","model":"VRT","rank_in_archive_order":1,"of":4,"metrics":{"PSNR":"36.01","SSIM":"0.9434"},"uses_additional_data":false},{"leaderboard":"/sota/video-denoising-on-davis-sigma10","task":"Video Denoising","dataset":"DAVIS sigma10","model":"VRT","rank_in_archive_order":2,"of":6,"metrics":{"PSNR":"40.82"},"uses_additional_data":false},{"leaderboard":"/sota/video-denoising-on-davis-sigma20","task":"Video Denoising","dataset":"DAVIS sigma20","model":"VRT","rank_in_archive_order":2,"of":7,"metrics":{"PSNR":"38.15"},"uses_additional_data":false},{"leaderboard":"/sota/video-denoising-on-davis-sigma30","task":"Video Denoising","dataset":"DAVIS sigma30","model":"VRT","rank_in_archive_order":3,"of":7,"metrics":{"PSNR":"36.52"},"uses_additional_data":false},{"leaderboard":"/sota/video-denoising-on-davis-sigma40","task":"Video Denoising","dataset":"DAVIS sigma40","model":"VRT","rank_in_archive_order":3,"of":8,"metrics":{"PSNR":"35.32"},"uses_additional_data":false},{"leaderboard":"/sota/video-denoising-on-davis-sigma50","task":"Video Denoising","dataset":"DAVIS sigma50","model":"VRT","rank_in_archive_order":3,"of":8,"metrics":{"PSNR":"34.36"},"uses_additional_data":false},{"leaderboard":"/sota/video-denoising-on-set8-sigma10","task":"Video Denoising","dataset":"Set8 sigma10","model":"VRT","rank_in_archive_order":2,"of":6,"metrics":{"PSNR":"37.88"},"uses_additional_data":false},{"leaderboard":"/sota/video-denoising-on-set8-sigma20","task":"Video Denoising","dataset":"Set8 sigma20","model":"VRT","rank_in_archive_order":2,"of":7,"metrics":{"PSNR":"35.02"},"uses_additional_data":false},{"leaderboard":"/sota/video-denoising-on-set8-sigma30","task":"Video Denoising","dataset":"Set8 sigma30","model":"VRT","rank_in_archive_order":2,"of":7,"metrics":{"PSNR":"33.35"},"uses_additional_data":false},{"leaderboard":"/sota/video-denoising-on-set8-sigma40","task":"Video Denoising","dataset":"Set8 sigma40","model":"VRT","rank_in_archive_order":3,"of":8,"metrics":{"PSNR":"32.15"},"uses_additional_data":false},{"leaderboard":"/sota/video-denoising-on-set8-sigma50","task":"Video Denoising","dataset":"Set8 sigma50","model":"VRT","rank_in_archive_order":3,"of":9,"metrics":{"PSNR":"31.22"},"uses_additional_data":false},{"leaderboard":"/sota/video-frame-interpolation-on-vid4-4x","task":"Video Frame Interpolation","dataset":"Vid4 - 4x upscaling","model":"VRT","rank_in_archive_order":1,"of":5,"metrics":{"PSNR":"27.46","Parameters":"4450000","SSIM":"0.8392"},"uses_additional_data":false},{"leaderboard":"/sota/video-super-resolution-on-msu-super-1","task":"Video Super-Resolution","dataset":"MSU Super-Resolution for Video Compression","model":"VRT + x264","rank_in_archive_order":17,"of":85,"metrics":{"BSQ-rate over ERQA":"1.578","BSQ-rate over LPIPS":"1.259","BSQ-rate over MS-SSIM":"0.662","BSQ-rate over PSNR":"1.09","BSQ-rate over Subjective Score":"1.245","BSQ-rate over VMAF":"0.7"},"uses_additional_data":false},{"leaderboard":"/sota/video-super-resolution-on-msu-super-1","task":"Video Super-Resolution","dataset":"MSU Super-Resolution for Video Compression","model":"VRT + x265","rank_in_archive_order":28,"of":85,"metrics":{"BSQ-rate over ERQA":"8.92","BSQ-rate over LPIPS":"11.329","BSQ-rate over MS-SSIM":"1.257","BSQ-rate over PSNR":"6.634","BSQ-rate over Subjective Score":"2.023","BSQ-rate over VMAF":"1.217"},"uses_additional_data":false},{"leaderboard":"/sota/video-super-resolution-on-msu-super-1","task":"Video Super-Resolution","dataset":"MSU Super-Resolution for Video Compression","model":"VRT + vvenc","rank_in_archive_order":29,"of":85,"metrics":{"BSQ-rate over ERQA":"18.333","BSQ-rate over LPIPS":"11.496","BSQ-rate over MS-SSIM":"0.836","BSQ-rate over PSNR":"5.777","BSQ-rate over Subjective Score":"2.235","BSQ-rate over VMAF":"0.652"},"uses_additional_data":false},{"leaderboard":"/sota/video-super-resolution-on-msu-super-1","task":"Video Super-Resolution","dataset":"MSU Super-Resolution for Video Compression","model":"VRT + uavs3e","rank_in_archive_order":33,"of":85,"metrics":{"BSQ-rate over ERQA":"6.619","BSQ-rate over LPIPS":"4.003","BSQ-rate over MS-SSIM":"1.982","BSQ-rate over PSNR":"5.862","BSQ-rate over Subjective Score":"2.511","BSQ-rate over VMAF":"1.425"},"uses_additional_data":false},{"leaderboard":"/sota/video-super-resolution-on-msu-super-1","task":"Video Super-Resolution","dataset":"MSU Super-Resolution for Video Compression","model":"VRT + aomenc","rank_in_archive_order":34,"of":85,"metrics":{"BSQ-rate over ERQA":"12.289","BSQ-rate over LPIPS":"4.429","BSQ-rate over MS-SSIM":"2.797","BSQ-rate over PSNR":"10.075","BSQ-rate over Subjective Score":"2.631","BSQ-rate over VMAF":"1.733"},"uses_additional_data":false},{"leaderboard":"/sota/video-super-resolution-on-msu-vsr-benchmark","task":"Video Super-Resolution","dataset":"MSU Video Super Resolution Benchmark: Detail Restoration","model":"VRT","rank_in_archive_order":1,"of":32,"metrics":{"1 - LPIPS":"0.929","ERQAv1.0":"0.758","FPS":"2.778","PSNR":"31.669","QRCRv1.0":"0.722","SSIM":"0.902","Subjective score":"7.628"},"uses_additional_data":false},{"leaderboard":"/sota/video-super-resolution-on-msu-video-upscalers","task":"Video Super-Resolution","dataset":"MSU Video Upscalers: Quality Enhancement","model":"VRT-Reds-L","rank_in_archive_order":29,"of":48,"metrics":{"LPIPS":"0.343","PSNR":"31.01","SSIM":"0.869"},"uses_additional_data":false},{"leaderboard":"/sota/video-super-resolution-on-udm10-4x-upscaling","task":"Video Super-Resolution","dataset":"UDM10 - 4x upscaling","model":"VRT","rank_in_archive_order":1,"of":7,"metrics":{"PSNR":"41.05","SSIM":"0.9737"},"uses_additional_data":false},{"leaderboard":"/sota/video-super-resolution-on-vid4-4x-upscaling","task":"Video Super-Resolution","dataset":"Vid4 - 4x upscaling","model":"VRT","rank_in_archive_order":8,"of":27,"metrics":{"PSNR":"27.93","SSIM":"0.8425"},"uses_additional_data":false},{"leaderboard":"/sota/video-super-resolution-on-vid4-4x-upscaling-1","task":"Video Super-Resolution","dataset":"Vid4 - 4x upscaling - BD degradation","model":"VRT","rank_in_archive_order":2,"of":18,"metrics":{"PSNR":"29.42","SSIM":"0.8795"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2201.12288","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2201.12288"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jingyunliang/vrt","reach":null}],"summary":{"ran_fixture":3,"ran_draft_wrong":1,"unverified":1},"by_repo_kind":{"official":{"samples":3,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":5,"samples":[{"code_sha256_prefix":"9ea4d5b8b4d5ef2f","entry":"drop_path","repo":"jingyunliang/vrt","repo_kind":"official","path":"models/network_vrt.py","file_url":"https://github.com/jingyunliang/vrt/blob/HEAD/models/network_vrt.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"9ea4d5b8b4d5ef2f"}},{"code_sha256_prefix":"c4d261acbb02d0e1","entry":"flow_warp","repo":"jingyunliang/vrt","repo_kind":"official","path":"models/network_vrt.py","file_url":"https://github.com/jingyunliang/vrt/blob/HEAD/models/network_vrt.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"c4d261acbb02d0e1"}},{"code_sha256_prefix":"a1cad82b9beff0c7","entry":"test_video","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"a1cad82b9beff0c7"}},{"code_sha256_prefix":"8347c57b16dab539","entry":"trunc_normal_","repo":"jingyunliang/vrt","repo_kind":"official","path":"models/network_vrt.py","file_url":"https://github.com/jingyunliang/vrt/blob/HEAD/models/network_vrt.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"8347c57b16dab539"}},{"code_sha256_prefix":"705758eeba9070e3","entry":"test_clip","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"705758eeba9070e3"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}