{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/autovfx-physically-realistic-video-editing","title":"AutoVFX: Physically Realistic Video Editing from Natural Language Instructions","arxiv_id":"2411.02394","date":"2024-11-04","proceeding":null,"authors":["Hao-Yu Hsu","Zhi-Hao Lin","Albert Zhai","Hongchi Xia","Shenlong Wang"],"abstract":"Modern visual effects (VFX) software has made it possible for skilled artists to create imagery of virtually anything. However, the creation process remains laborious, complex, and largely inaccessible to everyday users. In this work, we present AutoVFX, a framework that automatically creates realistic and dynamic VFX videos from a single video and natural language instructions. By carefully integrating neural scene modeling, LLM-based code generation, and physical simulation, AutoVFX is able to provide physically-grounded, photorealistic editing effects that can be controlled directly using natural language instructions. We conduct extensive experiments to validate AutoVFX's efficacy across a diverse spectrum of videos and instructions. Quantitative and qualitative results suggest that AutoVFX outperforms all competing methods by a large margin in generative quality, instruction alignment, editing versatility, and physical plausibility.","url_abs":"https://arxiv.org/abs/2411.02394v1","url_pdf":"https://arxiv.org/pdf/2411.02394v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"autovfx-physically-realistic-video-editing","repo_url":"https://github.com/haoyuhsu/autovfx","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"code-generation","task_name":"Code Generation"},{"task_slug":"video-editing","task_name":"Video Editing"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2411.02394","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2411.02394"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/haoyuhsu/autovfx","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":2,"unverified":13},"by_repo_kind":{"listed":{"samples":15,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"460e3a6383594a19","entry":"image_ids_to_pair_id","repo":"haoyuhsu/autovfx","repo_kind":"listed","path":"dataset_utils/database.py","file_url":"https://github.com/haoyuhsu/autovfx/blob/HEAD/dataset_utils/database.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"460e3a6383594a19"}},{"code_sha256_prefix":"e7dcd94ad331e47d","entry":"pair_id_to_image_ids","repo":"haoyuhsu/autovfx","repo_kind":"listed","path":"dataset_utils/database.py","file_url":"https://github.com/haoyuhsu/autovfx/blob/HEAD/dataset_utils/database.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e7dcd94ad331e47d"}},{"code_sha256_prefix":"141f61e2147729f4","entry":"array_to_blob","repo":"haoyuhsu/autovfx","repo_kind":"listed","path":"dataset_utils/database.py","file_url":"https://github.com/haoyuhsu/autovfx/blob/HEAD/dataset_utils/database.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"141f61e2147729f4"}},{"code_sha256_prefix":"34ff9b967b7c9fd8","entry":"check_mesh_path","repo":"haoyuhsu/autovfx","repo_kind":"listed","path":"gaussians_utils.py","file_url":"https://github.com/haoyuhsu/autovfx/blob/HEAD/gaussians_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"34ff9b967b7c9fd8"}},{"code_sha256_prefix":"607179b33c25da2d","entry":"create_camera_list","repo":"haoyuhsu/autovfx","repo_kind":"listed","path":"blender/all_rendering.py","file_url":"https://github.com/haoyuhsu/autovfx/blob/HEAD/blender/all_rendering.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"607179b33c25da2d"}},{"code_sha256_prefix":"2b622c63eeed9fb1","entry":"downsample_image","repo":"haoyuhsu/autovfx","repo_kind":"listed","path":"blender/blend_all.py","file_url":"https://github.com/haoyuhsu/autovfx/blob/HEAD/blender/blend_all.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2b622c63eeed9fb1"}},{"code_sha256_prefix":"103bafd937e08be9","entry":"load_depth","repo":"haoyuhsu/autovfx","repo_kind":"listed","path":"blender/blend_all.py","file_url":"https://github.com/haoyuhsu/autovfx/blob/HEAD/blender/blend_all.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"103bafd937e08be9"}},{"code_sha256_prefix":"7f0acca1a3f53b83","entry":"load_rgb","repo":"haoyuhsu/autovfx","repo_kind":"listed","path":"blender/blend_all.py","file_url":"https://github.com/haoyuhsu/autovfx/blob/HEAD/blender/blend_all.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7f0acca1a3f53b83"}},{"code_sha256_prefix":"f325cf8bb45ce2a6","entry":"matrix_to_quaternion","repo":"haoyuhsu/autovfx","repo_kind":"listed","path":"rotation_utils.py","file_url":"https://github.com/haoyuhsu/autovfx/blob/HEAD/rotation_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f325cf8bb45ce2a6"}},{"code_sha256_prefix":"c158260ba633e27a","entry":"quaternion_multiply","repo":"haoyuhsu/autovfx","repo_kind":"listed","path":"rotation_utils.py","file_url":"https://github.com/haoyuhsu/autovfx/blob/HEAD/rotation_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c158260ba633e27a"}},{"code_sha256_prefix":"547fb2aab836b2a8","entry":"quternion_to_matrix","repo":"haoyuhsu/autovfx","repo_kind":"listed","path":"rotation_utils.py","file_url":"https://github.com/haoyuhsu/autovfx/blob/HEAD/rotation_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"547fb2aab836b2a8"}},{"code_sha256_prefix":"1a7c58219c1ce58e","entry":"read_cameras_binary","repo":"haoyuhsu/autovfx","repo_kind":"listed","path":"dataset_utils/colmap_read_model.py","file_url":"https://github.com/haoyuhsu/autovfx/blob/HEAD/dataset_utils/colmap_read_model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1a7c58219c1ce58e"}},{"code_sha256_prefix":"2de01ac9135d19ec","entry":"read_cameras_text","repo":"haoyuhsu/autovfx","repo_kind":"listed","path":"dataset_utils/colmap_read_model.py","file_url":"https://github.com/haoyuhsu/autovfx/blob/HEAD/dataset_utils/colmap_read_model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2de01ac9135d19ec"}},{"code_sha256_prefix":"14e96def7cba1fa1","entry":"read_next_bytes","repo":"haoyuhsu/autovfx","repo_kind":"listed","path":"dataset_utils/colmap_read_model.py","file_url":"https://github.com/haoyuhsu/autovfx/blob/HEAD/dataset_utils/colmap_read_model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"14e96def7cba1fa1"}},{"code_sha256_prefix":"7cbef8846fef7bf2","entry":"sample_point_on_sphere","repo":"haoyuhsu/autovfx","repo_kind":"listed","path":"blender/asset_rendering.py","file_url":"https://github.com/haoyuhsu/autovfx/blob/HEAD/blender/asset_rendering.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7cbef8846fef7bf2"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}