{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/real3d-portrait-one-shot-realistic-3d-talking","title":"Real3D-Portrait: One-shot Realistic 3D Talking Portrait Synthesis","arxiv_id":"2401.08503","date":"2024-01-16","proceeding":null,"authors":["Zhenhui Ye","Tianyun Zhong","Yi Ren","Jiaqi Yang","Weichuang Li","Jiawei Huang","Ziyue Jiang","Jinzheng He","Rongjie Huang","Jinglin Liu","Chen Zhang","Xiang Yin","Zejun Ma","Zhou Zhao"],"abstract":"One-shot 3D talking portrait generation aims to reconstruct a 3D avatar from an unseen image, and then animate it with a reference video or audio to generate a talking portrait video. The existing methods fail to simultaneously achieve the goals of accurate 3D avatar reconstruction and stable talking face animation. Besides, while the existing works mainly focus on synthesizing the head part, it is also vital to generate natural torso and background segments to obtain a realistic talking portrait video. To address these limitations, we present Real3D-Potrait, a framework that (1) improves the one-shot 3D reconstruction power with a large image-to-plane model that distills 3D prior knowledge from a 3D face generative model; (2) facilitates accurate motion-conditioned animation with an efficient motion adapter; (3) synthesizes realistic video with natural torso movement and switchable background using a head-torso-background super-resolution model; and (4) supports one-shot audio-driven talking face generation with a generalizable audio-to-motion model. Extensive experiments show that Real3D-Portrait generalizes well to unseen identities and generates more realistic talking portrait videos compared to previous methods. Video samples and source code are available at https://real3dportrait.github.io .","url_abs":"https://arxiv.org/abs/2401.08503v3","url_pdf":"https://arxiv.org/pdf/2401.08503v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"real3d-portrait-one-shot-realistic-3d-talking","repo_url":"https://github.com/yerfor/Real3DPortrait","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"3d-reconstruction","task_name":"3D Reconstruction"},{"task_slug":"face-generation","task_name":"Face Generation"},{"task_slug":"super-resolution","task_name":"Super-Resolution"},{"task_slug":"talking-face-generation","task_name":"Talking Face Generation"}],"methods":[{"method_slug":"focus","method_name":"Focus"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2401.08503","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.08503"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yerfor/Real3DPortrait","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":5,"unverified":1},"by_repo_kind":{"official":{"samples":6,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"0f6c5532e66c554c","entry":"filter_state_dict","repo":"yerfor/Real3DPortrait","repo_kind":"official","path":"deep_3drecon/deep_3drecon_models/networks.py","file_url":"https://github.com/yerfor/Real3DPortrait/blob/HEAD/deep_3drecon/deep_3drecon_models/networks.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0f6c5532e66c554c"}},{"code_sha256_prefix":"962cf09d55becbb1","entry":"get_scheduler","repo":"yerfor/Real3DPortrait","repo_kind":"official","path":"deep_3drecon/deep_3drecon_models/networks.py","file_url":"https://github.com/yerfor/Real3DPortrait/blob/HEAD/deep_3drecon/deep_3drecon_models/networks.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"962cf09d55becbb1"}},{"code_sha256_prefix":"945b4307eb32c770","entry":"perceptual_loss","repo":"yerfor/Real3DPortrait","repo_kind":"official","path":"deep_3drecon/deep_3drecon_models/losses.py","file_url":"https://github.com/yerfor/Real3DPortrait/blob/HEAD/deep_3drecon/deep_3drecon_models/losses.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"945b4307eb32c770"}},{"code_sha256_prefix":"2e9ce6a02832f7c8","entry":"perspective_projection","repo":"yerfor/Real3DPortrait","repo_kind":"official","path":"deep_3drecon/deep_3drecon_models/bfm.py","file_url":"https://github.com/yerfor/Real3DPortrait/blob/HEAD/deep_3drecon/deep_3drecon_models/bfm.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2e9ce6a02832f7c8"}},{"code_sha256_prefix":"8ee1568e301607cf","entry":"photo_loss","repo":"yerfor/Real3DPortrait","repo_kind":"official","path":"deep_3drecon/deep_3drecon_models/losses.py","file_url":"https://github.com/yerfor/Real3DPortrait/blob/HEAD/deep_3drecon/deep_3drecon_models/losses.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8ee1568e301607cf"}},{"code_sha256_prefix":"2870025ec65fec68","entry":"resize_n_crop","repo":"yerfor/Real3DPortrait","repo_kind":"official","path":"deep_3drecon/deep_3drecon_models/losses.py","file_url":"https://github.com/yerfor/Real3DPortrait/blob/HEAD/deep_3drecon/deep_3drecon_models/losses.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2870025ec65fec68"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}