{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/tarvis-a-unified-approach-for-target-based","title":"TarViS: A Unified Approach for Target-based Video Segmentation","arxiv_id":"2301.02657","date":"2023-01-06","proceeding":"CVPR 2023 1","authors":["Ali Athar","Alexander Hermans","Jonathon Luiten","Deva Ramanan","Bastian Leibe"],"abstract":"The general domain of video segmentation is currently fragmented into different tasks spanning multiple benchmarks. Despite rapid progress in the state-of-the-art, current methods are overwhelmingly task-specific and cannot conceptually generalize to other tasks. Inspired by recent approaches with multi-task capability, we propose TarViS: a novel, unified network architecture that can be applied to any task that requires segmenting a set of arbitrarily defined 'targets' in video. Our approach is flexible with respect to how tasks define these targets, since it models the latter as abstract 'queries' which are then used to predict pixel-precise target masks. A single TarViS model can be trained jointly on a collection of datasets spanning different tasks, and can hot-swap between tasks during inference without any task-specific retraining. To demonstrate its effectiveness, we apply TarViS to four different tasks, namely Video Instance Segmentation (VIS), Video Panoptic Segmentation (VPS), Video Object Segmentation (VOS) and Point Exemplar-guided Tracking (PET). Our unified, jointly trained model achieves state-of-the-art performance on 5/7 benchmarks spanning these four tasks, and competitive performance on the remaining two. Code and model weights are available at: https://github.com/Ali2500/TarViS","url_abs":"https://arxiv.org/abs/2301.02657v2","url_pdf":"https://arxiv.org/pdf/2301.02657v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"tarvis-a-unified-approach-for-target-based","repo_url":"https://github.com/Ali2500/TarViS","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"instance-segmentation","task_name":"Instance Segmentation"},{"task_slug":"panoptic-segmentation","task_name":"Panoptic Segmentation"},{"task_slug":"segmentation","task_name":"Segmentation"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"},{"task_slug":"video-instance-segmentation","task_name":"Video Instance Segmentation"},{"task_slug":"video-object-segmentation","task_name":"Video Object Segmentation"},{"task_slug":"video-panoptic-segmentation","task_name":"Video Panoptic Segmentation"},{"task_slug":"video-segmentation","task_name":"Video Segmentation"},{"task_slug":"video-semantic-segmentation","task_name":"Video Semantic Segmentation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/visual-object-tracking-on-davis-2017","task":"Semi-Supervised Video Object Segmentation","dataset":"DAVIS 2017 (val)","model":"TarViS","rank_in_archive_order":24,"of":81,"metrics":{"F-measure (Mean)":"88.5","J&F":"85.3","Jaccard (Mean)":"81.7"},"uses_additional_data":true},{"leaderboard":"/sota/video-instance-segmentation-on-ovis-1","task":"Video Instance Segmentation","dataset":"OVIS validation","model":"TarViS (Swin-L)","rank_in_archive_order":14,"of":44,"metrics":{"AP50":"67.8","AP75":"44.6","AR1":"18.0","AR10":"50.4","mask AP":"43.2"},"uses_additional_data":true},{"leaderboard":"/sota/video-instance-segmentation-on-ovis-1","task":"Video Instance Segmentation","dataset":"OVIS validation","model":"TarViS  (Swin-T)","rank_in_archive_order":27,"of":44,"metrics":{"AP50":"55.0","AP75":"34.4","AR1":"16.1","AR10":"40.9","mask AP":"34.0"},"uses_additional_data":true},{"leaderboard":"/sota/video-instance-segmentation-on-ovis-1","task":"Video Instance Segmentation","dataset":"OVIS validation","model":"TarViS (ResNet-50)","rank_in_archive_order":29,"of":44,"metrics":{"AP50":"52.5","AP75":"30.4","AR1":"15.9","AR10":"39.9","mask AP":"31.1"},"uses_additional_data":true},{"leaderboard":"/sota/video-instance-segmentation-on-youtube-vis-2","task":"Video Instance Segmentation","dataset":"YouTube-VIS 2021","model":"TarViS (Swin-L)","rank_in_archive_order":7,"of":26,"metrics":{"AP50":"81.4","AP75":"67.6","AR1":"47.6","AR10":"64.8","mask AP":"60.2"},"uses_additional_data":true},{"leaderboard":"/sota/video-instance-segmentation-on-youtube-vis-2","task":"Video Instance Segmentation","dataset":"YouTube-VIS 2021","model":"TarViS (Swin-T)","rank_in_archive_order":20,"of":26,"metrics":{"AP50":"71.6","AP75":"56.6","AR1":"42.2","AR10":"57.2","mask AP":"50.9"},"uses_additional_data":true},{"leaderboard":"/sota/video-instance-segmentation-on-youtube-vis-2","task":"Video Instance Segmentation","dataset":"YouTube-VIS 2021","model":"TarViS (ResNet-50)","rank_in_archive_order":22,"of":26,"metrics":{"AP50":"69.6","AP75":"53.2","AR1":"40.5","AR10":"55.9","mask AP":"48.3"},"uses_additional_data":true},{"leaderboard":"/sota/video-panoptic-segmentation-on-cityscapes-vps","task":"Video Panoptic Segmentation","dataset":"Cityscapes-VPS","model":"TarViS (Swin-L)","rank_in_archive_order":4,"of":8,"metrics":{"VPQ":"58.9","VPQ (stuff)":"69.9","VPQ (thing)":"43.7"},"uses_additional_data":true},{"leaderboard":"/sota/video-panoptic-segmentation-on-cityscapes-vps","task":"Video Panoptic Segmentation","dataset":"Cityscapes-VPS","model":"TarViS (Swin-T)","rank_in_archive_order":5,"of":8,"metrics":{"VPQ":"58.0","VPQ (stuff)":"69.0","VPQ (thing)":"42.9"},"uses_additional_data":true},{"leaderboard":"/sota/video-panoptic-segmentation-on-cityscapes-vps","task":"Video Panoptic Segmentation","dataset":"Cityscapes-VPS","model":"TarViS (ResNet-50)","rank_in_archive_order":8,"of":8,"metrics":{"VPQ":"53.3","VPQ (stuff)":"66.0","VPQ (thing)":"35.9"},"uses_additional_data":true},{"leaderboard":"/sota/video-panoptic-segmentation-on-kitti-step","task":"Video Panoptic Segmentation","dataset":"KITTI-STEP","model":"TarViS (Swin-L)","rank_in_archive_order":2,"of":6,"metrics":{"AQ":"72.0","SQ":"72.0","STQ":"73.0"},"uses_additional_data":true},{"leaderboard":"/sota/video-panoptic-segmentation-on-kitti-step","task":"Video Panoptic Segmentation","dataset":"KITTI-STEP","model":"TarViS (Swin-T)","rank_in_archive_order":3,"of":6,"metrics":{"AQ":"71.2","SQ":"69.9","STQ":"70.6"},"uses_additional_data":true},{"leaderboard":"/sota/video-panoptic-segmentation-on-kitti-step","task":"Video Panoptic Segmentation","dataset":"KITTI-STEP","model":"TarViS (ResNet-50)","rank_in_archive_order":4,"of":6,"metrics":{"AQ":"70.3","SQ":"68.8","STQ":"69.6"},"uses_additional_data":true},{"leaderboard":"/sota/video-panoptic-segmentation-on-vipseg","task":"Video Panoptic Segmentation","dataset":"VIPSeg","model":"TarViS (Swin-L)","rank_in_archive_order":9,"of":12,"metrics":{"STQ":"52.9","VPQ":"48.0"},"uses_additional_data":true},{"leaderboard":"/sota/video-panoptic-segmentation-on-vipseg","task":"Video Panoptic Segmentation","dataset":"VIPSeg","model":"TarViS (Swin-T)","rank_in_archive_order":11,"of":12,"metrics":{"STQ":"45.3","VPQ":"35.8"},"uses_additional_data":true},{"leaderboard":"/sota/video-panoptic-segmentation-on-vipseg","task":"Video Panoptic Segmentation","dataset":"VIPSeg","model":"TarViS (ResNet-50)","rank_in_archive_order":12,"of":12,"metrics":{"STQ":"43.1","VPQ":"33.5"},"uses_additional_data":true}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2301.02657","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2301.02657"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/Ali2500/TarViS","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":6},"by_repo_kind":{"official":{"samples":6,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"723587c9b01223c1","entry":"cat","repo":"Ali2500/TarViS","repo_kind":"official","path":"tarvis/modelling/point_sampling_utils.py","file_url":"https://github.com/Ali2500/TarViS/blob/HEAD/tarvis/modelling/point_sampling_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"723587c9b01223c1"}},{"code_sha256_prefix":"ca3f23e7a9f1011a","entry":"compute_padded_dims","repo":"Ali2500/TarViS","repo_kind":"official","path":"tarvis/modelling/utils.py","file_url":"https://github.com/Ali2500/TarViS/blob/HEAD/tarvis/modelling/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ca3f23e7a9f1011a"}},{"code_sha256_prefix":"3117cc9941298e0c","entry":"compute_padding","repo":"Ali2500/TarViS","repo_kind":"official","path":"tarvis/modelling/utils.py","file_url":"https://github.com/Ali2500/TarViS/blob/HEAD/tarvis/modelling/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3117cc9941298e0c"}},{"code_sha256_prefix":"35425cfa0d3f34a7","entry":"point_sample","repo":"Ali2500/TarViS","repo_kind":"official","path":"tarvis/modelling/point_sampling_utils.py","file_url":"https://github.com/Ali2500/TarViS/blob/HEAD/tarvis/modelling/point_sampling_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"35425cfa0d3f34a7"}},{"code_sha256_prefix":"a29da2f7b8a01b44","entry":"point_sample_3d","repo":"Ali2500/TarViS","repo_kind":"official","path":"tarvis/modelling/point_sampling_utils.py","file_url":"https://github.com/Ali2500/TarViS/blob/HEAD/tarvis/modelling/point_sampling_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a29da2f7b8a01b44"}},{"code_sha256_prefix":"70f1b932a11d5437","entry":"split_by_query_group","repo":"Ali2500/TarViS","repo_kind":"official","path":"tarvis/modelling/utils.py","file_url":"https://github.com/Ali2500/TarViS/blob/HEAD/tarvis/modelling/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"70f1b932a11d5437"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}