{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/depth-any-video-with-scalable-synthetic-data","title":"Depth Any Video with Scalable Synthetic Data","arxiv_id":"2410.10815","date":"2024-10-14","proceeding":null,"authors":["Honghui Yang","Di Huang","Wei Yin","Chunhua Shen","Haifeng Liu","Xiaofei He","Binbin Lin","Wanli Ouyang","Tong He"],"abstract":"Video depth estimation has long been hindered by the scarcity of consistent and scalable ground truth data, leading to inconsistent and unreliable results. In this paper, we introduce Depth Any Video, a model that tackles the challenge through two key innovations. First, we develop a scalable synthetic data pipeline, capturing real-time video depth data from diverse synthetic environments, yielding 40,000 video clips of 5-second duration, each with precise depth annotations. Second, we leverage the powerful priors of generative video diffusion models to handle real-world videos effectively, integrating advanced techniques such as rotary position encoding and flow matching to further enhance flexibility and efficiency. Unlike previous models, which are limited to fixed-length video sequences, our approach introduces a novel mixed-duration training strategy that handles videos of varying lengths and performs robustly across different frame rates-even on single frames. At inference, we propose a depth interpolation method that enables our model to infer high-resolution video depth across sequences of up to 150 frames. Our model outperforms all previous generative depth models in terms of spatial accuracy and temporal consistency.","url_abs":"https://arxiv.org/abs/2410.10815v1","url_pdf":"https://arxiv.org/pdf/2410.10815v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"depth-any-video-with-scalable-synthetic-data","repo_url":"https://github.com/Nightmare-n/DepthAnyVideo","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}}],"tasks":[{"task_slug":"depth-estimation","task_name":"Depth Estimation"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2410.10815","atlas_url":"https://app.syntology.ai/?focus=2410.10815","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.10815"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Nightmare-n/DepthAnyVideo","reach":{"status":"ok","spdx":"NOASSERTION"}}],"summary":{"ran":8,"unverified":4},"by_repo_kind":{"official":{"samples":12,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":12,"samples":[{"code_sha256_prefix":"dda1836513498034","entry":"colorize_depth","repo":"Nightmare-n/DepthAnyVideo","repo_kind":"official","path":"dav/pipelines/dav_pipeline.py","file_url":"https://github.com/Nightmare-n/DepthAnyVideo/blob/HEAD/dav/pipelines/dav_pipeline.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"dda1836513498034"}},{"code_sha256_prefix":"9de6198eba9e20ac","entry":"crop","repo":"Nightmare-n/DepthAnyVideo","repo_kind":"official","path":"dav/utils/img_utils.py","file_url":"https://github.com/Nightmare-n/DepthAnyVideo/blob/HEAD/dav/utils/img_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"9de6198eba9e20ac"}},{"code_sha256_prefix":"eacc04c276c927c2","entry":"d1","repo":"Nightmare-n/DepthAnyVideo","repo_kind":"official","path":"dav/utils/eval_utils.py","file_url":"https://github.com/Nightmare-n/DepthAnyVideo/blob/HEAD/dav/utils/eval_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"eacc04c276c927c2"}},{"code_sha256_prefix":"a3878270fa1ba21f","entry":"d2","repo":"Nightmare-n/DepthAnyVideo","repo_kind":"official","path":"dav/utils/eval_utils.py","file_url":"https://github.com/Nightmare-n/DepthAnyVideo/blob/HEAD/dav/utils/eval_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"a3878270fa1ba21f"}},{"code_sha256_prefix":"f8c42aeb933a366a","entry":"d3","repo":"Nightmare-n/DepthAnyVideo","repo_kind":"official","path":"dav/utils/eval_utils.py","file_url":"https://github.com/Nightmare-n/DepthAnyVideo/blob/HEAD/dav/utils/eval_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"f8c42aeb933a366a"}},{"code_sha256_prefix":"f164270170de7c55","entry":"get_2d_sincos_pos_embed","repo":"Nightmare-n/DepthAnyVideo","repo_kind":"official","path":"dav/models/embeddings.py","file_url":"https://github.com/Nightmare-n/DepthAnyVideo/blob/HEAD/dav/models/embeddings.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"f164270170de7c55"}},{"code_sha256_prefix":"092ac9fec96772b3","entry":"imresize_max","repo":"Nightmare-n/DepthAnyVideo","repo_kind":"official","path":"dav/utils/img_utils.py","file_url":"https://github.com/Nightmare-n/DepthAnyVideo/blob/HEAD/dav/utils/img_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"092ac9fec96772b3"}},{"code_sha256_prefix":"f62dbc206ca86a39","entry":"resize","repo":"Nightmare-n/DepthAnyVideo","repo_kind":"official","path":"dav/utils/img_utils.py","file_url":"https://github.com/Nightmare-n/DepthAnyVideo/blob/HEAD/dav/utils/img_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"f62dbc206ca86a39"}},{"code_sha256_prefix":"59fa7b38de05f443","entry":"get_2d_sincos_pos_embed_from_grid","repo":"Nightmare-n/DepthAnyVideo","repo_kind":"official","path":"dav/models/embeddings.py","file_url":"https://github.com/Nightmare-n/DepthAnyVideo/blob/HEAD/dav/models/embeddings.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"59fa7b38de05f443"}},{"code_sha256_prefix":"3e3689acce2d5915","entry":"get_down_block","repo":"Nightmare-n/DepthAnyVideo","repo_kind":"official","path":"dav/models/unets/unet_3d_rope_blocks.py","file_url":"https://github.com/Nightmare-n/DepthAnyVideo/blob/HEAD/dav/models/unets/unet_3d_rope_blocks.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"3e3689acce2d5915"}},{"code_sha256_prefix":"38f69a3d1e46016d","entry":"get_timestep_embedding","repo":"Nightmare-n/DepthAnyVideo","repo_kind":"official","path":"dav/models/embeddings.py","file_url":"https://github.com/Nightmare-n/DepthAnyVideo/blob/HEAD/dav/models/embeddings.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"38f69a3d1e46016d"}},{"code_sha256_prefix":"17a44e9d5e9ba18e","entry":"get_up_block","repo":"Nightmare-n/DepthAnyVideo","repo_kind":"official","path":"dav/models/unets/unet_3d_rope_blocks.py","file_url":"https://github.com/Nightmare-n/DepthAnyVideo/blob/HEAD/dav/models/unets/unet_3d_rope_blocks.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"17a44e9d5e9ba18e"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}