{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/aniportrait-audio-driven-synthesis-of","title":"AniPortrait: Audio-Driven Synthesis of Photorealistic Portrait Animation","arxiv_id":"2403.17694","date":"2024-03-26","proceeding":null,"authors":["Huawei Wei","Zejun Yang","Zhisheng Wang"],"abstract":"In this study, we propose AniPortrait, a novel framework for generating high-quality animation driven by audio and a reference portrait image. Our methodology is divided into two stages. Initially, we extract 3D intermediate representations from audio and project them into a sequence of 2D facial landmarks. Subsequently, we employ a robust diffusion model, coupled with a motion module, to convert the landmark sequence into photorealistic and temporally consistent portrait animation. Experimental results demonstrate the superiority of AniPortrait in terms of facial naturalness, pose diversity, and visual quality, thereby offering an enhanced perceptual experience. Moreover, our methodology exhibits considerable potential in terms of flexibility and controllability, which can be effectively applied in areas such as facial motion editing or face reenactment. We release code and model weights at https://github.com/scutzzj/AniPortrait","url_abs":"https://arxiv.org/abs/2403.17694v1","url_pdf":"https://arxiv.org/pdf/2403.17694v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"aniportrait-audio-driven-synthesis-of","repo_url":"https://github.com/scutzzj/aniportrait","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"aniportrait-audio-driven-synthesis-of","repo_url":"https://github.com/zejun-yang/aniportrait","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"diversity","task_name":"Diversity"},{"task_slug":"face-reenactment","task_name":"Face Reenactment"},{"task_slug":"portrait-animation","task_name":"Portrait Animation"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2403.17694","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.17694"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/scutzzj/aniportrait","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/zejun-yang/aniportrait","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":5,"ran_draft_wrong":1,"unverified":5},"by_repo_kind":{"official":{"samples":11,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"3c6f442fbc45ccb4","entry":"collate_fn","repo":"scutzzj/aniportrait","repo_kind":"official","path":"src/dataset/dataset_face.py","file_url":"https://github.com/scutzzj/aniportrait/blob/HEAD/src/dataset/dataset_face.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3c6f442fbc45ccb4"}},{"code_sha256_prefix":"3c0feaae421deaa6","entry":"get_mask_from_lengths","repo":"scutzzj/aniportrait","repo_kind":"official","path":"src/audio_models/torch_utils.py","file_url":"https://github.com/scutzzj/aniportrait/blob/HEAD/src/audio_models/torch_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3c0feaae421deaa6"}},{"code_sha256_prefix":"aa7ccd5a0450ec8f","entry":"init_biased_mask","repo":"scutzzj/aniportrait","repo_kind":"official","path":"src/audio_models/pose_model.py","file_url":"https://github.com/scutzzj/aniportrait/blob/HEAD/src/audio_models/pose_model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"aa7ccd5a0450ec8f"}},{"code_sha256_prefix":"4007212e0e5fa312","entry":"linear_interpolation","repo":"scutzzj/aniportrait","repo_kind":"official","path":"src/audio_models/torch_utils.py","file_url":"https://github.com/scutzzj/aniportrait/blob/HEAD/src/audio_models/torch_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4007212e0e5fa312"}},{"code_sha256_prefix":"e8af53cde1de7240","entry":"mish","repo":"scutzzj/aniportrait","repo_kind":"official","path":"src/audio_models/mish.py","file_url":"https://github.com/scutzzj/aniportrait/blob/HEAD/src/audio_models/mish.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e8af53cde1de7240"}},{"code_sha256_prefix":"4719c763c53be3fe","entry":"zero_module","repo":"scutzzj/aniportrait","repo_kind":"official","path":"src/models/motion_module.py","file_url":"https://github.com/scutzzj/aniportrait/blob/HEAD/src/models/motion_module.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4719c763c53be3fe"}},{"code_sha256_prefix":"16309f96889a07a1","entry":"compute_snr","repo":"scutzzj/aniportrait","repo_kind":"official","path":"train_stage_1.py","file_url":"https://github.com/scutzzj/aniportrait/blob/HEAD/train_stage_1.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"16309f96889a07a1"}},{"code_sha256_prefix":"2c02b3b459122f59","entry":"decode_latents","repo":"scutzzj/aniportrait","repo_kind":"official","path":"train_stage_2.py","file_url":"https://github.com/scutzzj/aniportrait/blob/HEAD/train_stage_2.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2c02b3b459122f59"}},{"code_sha256_prefix":"fd4cbd132a7b01e2","entry":"enc_dec_mask","repo":"scutzzj/aniportrait","repo_kind":"official","path":"src/audio_models/pose_model.py","file_url":"https://github.com/scutzzj/aniportrait/blob/HEAD/src/audio_models/pose_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fd4cbd132a7b01e2"}},{"code_sha256_prefix":"525d02f457ec7c07","entry":"get_motion_module","repo":"scutzzj/aniportrait","repo_kind":"official","path":"src/models/motion_module.py","file_url":"https://github.com/scutzzj/aniportrait/blob/HEAD/src/models/motion_module.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"525d02f457ec7c07"}},{"code_sha256_prefix":"e55f24470648724c","entry":"torch_dfs","repo":"scutzzj/aniportrait","repo_kind":"official","path":"src/models/mutual_self_attention.py","file_url":"https://github.com/scutzzj/aniportrait/blob/HEAD/src/models/mutual_self_attention.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e55f24470648724c"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}