{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/tap-vid-a-benchmark-for-tracking-any-point-in","title":"TAP-Vid: A Benchmark for Tracking Any Point in a Video","arxiv_id":"2211.03726","date":"2022-11-07","proceeding":null,"authors":["Carl Doersch","Ankush Gupta","Larisa Markeeva","Adrià Recasens","Lucas Smaira","Yusuf Aytar","João Carreira","Andrew Zisserman","Yi Yang"],"abstract":"Generic motion understanding from video involves not only tracking objects, but also perceiving how their surfaces deform and move. This information is useful to make inferences about 3D shape, physical properties and object interactions. While the problem of tracking arbitrary physical points on surfaces over longer video clips has received some attention, no dataset or benchmark for evaluation existed, until now. In this paper, we first formalize the problem, naming it tracking any point (TAP). We introduce a companion benchmark, TAP-Vid, which is composed of both real-world videos with accurate human annotations of point tracks, and synthetic videos with perfect ground-truth point tracks. Central to the construction of our benchmark is a novel semi-automatic crowdsourced pipeline which uses optical flow estimates to compensate for easier, short-term motion like camera shake, allowing annotators to focus on harder sections of video. We validate our pipeline on synthetic data and propose a simple end-to-end point tracking model TAP-Net, showing that it outperforms all prior methods on our benchmark when trained on synthetic data.","url_abs":"https://arxiv.org/abs/2211.03726v2","url_pdf":"https://arxiv.org/pdf/2211.03726v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"tap-vid-a-benchmark-for-tracking-any-point-in","repo_url":"https://github.com/deepmind/tapnet","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"jax","reach":null},{"paper_slug":"tap-vid-a-benchmark-for-tracking-any-point-in","repo_url":"https://github.com/deepmind/perception_test","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"tap-vid-a-benchmark-for-tracking-any-point-in","repo_url":"https://github.com/google-deepmind/perception_test","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"optical-flow-estimation","task_name":"Optical Flow Estimation"},{"task_slug":"point-tracking","task_name":"Point Tracking"}],"methods":[],"datasets_introduced":[{"slug":"tap-vid","name":"TAP-Vid","full_name":""}],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2211.03726","atlas_url":"https://app.syntology.ai/?focus=2211.03726","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.03726"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/deepmind/tapnet","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/deepmind/perception_test","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/google-deepmind/perception_test","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":2,"ran_fixture":2,"unverified":8},"by_repo_kind":{"official":{"samples":10,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":2,"samples":[{"code_sha256_prefix":"e4b5cfd7b23f5b8a","entry":"FeatureGrids","repo":"deepmind/tapnet","repo_kind":"official","path":"tapnet/models/tapir_model.py","file_url":"https://github.com/deepmind/tapnet/blob/HEAD/tapnet/models/tapir_model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e4b5cfd7b23f5b8a"}},{"code_sha256_prefix":"fa3e94c3384e2433","entry":"QueryFeatures","repo":"deepmind/tapnet","repo_kind":"official","path":"tapnet/models/tapir_model.py","file_url":"https://github.com/deepmind/tapnet/blob/HEAD/tapnet/models/tapir_model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fa3e94c3384e2433"}},{"code_sha256_prefix":"76eb27227d072d59","entry":"construct_patch_kernel","repo":"deepmind/tapnet","repo_kind":"official","path":"tapnet/models/tapir_model.py","file_url":"https://github.com/deepmind/tapnet/blob/HEAD/tapnet/models/tapir_model.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"76eb27227d072d59"}},{"code_sha256_prefix":"e4c82c2698b56558","entry":"is_same_res","repo":"deepmind/tapnet","repo_kind":"official","path":"tapnet/models/tapir_model.py","file_url":"https://github.com/deepmind/tapnet/blob/HEAD/tapnet/models/tapir_model.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e4c82c2698b56558"}},{"code_sha256_prefix":"7a43ef7e87e76c0e","entry":"ExtraConvs","repo":"deepmind/tapnet","repo_kind":"official","path":"tapnet/models/tapir_model.py","file_url":"https://github.com/deepmind/tapnet/blob/HEAD/tapnet/models/tapir_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7a43ef7e87e76c0e"}},{"code_sha256_prefix":"2964dce1f0e378ea","entry":"PIPSMLPMixer","repo":"deepmind/tapnet","repo_kind":"official","path":"tapnet/models/tapir_model.py","file_url":"https://github.com/deepmind/tapnet/blob/HEAD/tapnet/models/tapir_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2964dce1f0e378ea"}},{"code_sha256_prefix":"95430143ca6992aa","entry":"PIPsConvBlock","repo":"deepmind/tapnet","repo_kind":"official","path":"tapnet/models/tapir_model.py","file_url":"https://github.com/deepmind/tapnet/blob/HEAD/tapnet/models/tapir_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"95430143ca6992aa"}},{"code_sha256_prefix":"58ee3b8b11f77a20","entry":"TAPIR","repo":"deepmind/tapnet","repo_kind":"official","path":"tapnet/models/tapir_model.py","file_url":"https://github.com/deepmind/tapnet/blob/HEAD/tapnet/models/tapir_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"58ee3b8b11f77a20"}},{"code_sha256_prefix":"fa8999c2e2fb539f","entry":"conv_channels_mixer","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"fa8999c2e2fb539f"}},{"code_sha256_prefix":"6822cb967fad105f","entry":"depthwise_conv_residual","repo":"deepmind/tapnet","repo_kind":"official","path":"tapnet/models/tapir_model.py","file_url":"https://github.com/deepmind/tapnet/blob/HEAD/tapnet/models/tapir_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6822cb967fad105f"}},{"code_sha256_prefix":"10fc62dae41762d7","entry":"extract_patch_depthwise_conv","repo":"deepmind/tapnet","repo_kind":"official","path":"tapnet/models/tapir_model.py","file_url":"https://github.com/deepmind/tapnet/blob/HEAD/tapnet/models/tapir_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"10fc62dae41762d7"}},{"code_sha256_prefix":"23e304fbbc8fabfe","entry":"layernorm","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"23e304fbbc8fabfe"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}