{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/identity-preserving-talking-face-generation","title":"Identity-Preserving Talking Face Generation with Landmark and Appearance Priors","arxiv_id":"2305.08293","date":"2023-05-15","proceeding":"CVPR 2023 1","authors":["Weizhi Zhong","Chaowei Fang","Yinqi Cai","Pengxu Wei","Gangming Zhao","Liang Lin","Guanbin Li"],"abstract":"Generating talking face videos from audio attracts lots of research interest. A few person-specific methods can generate vivid videos but require the target speaker's videos for training or fine-tuning. Existing person-generic methods have difficulty in generating realistic and lip-synced videos while preserving identity information. To tackle this problem, we propose a two-stage framework consisting of audio-to-landmark generation and landmark-to-video rendering procedures. First, we devise a novel Transformer-based landmark generator to infer lip and jaw landmarks from the audio. Prior landmark characteristics of the speaker's face are employed to make the generated landmarks coincide with the facial outline of the speaker. Then, a video rendering model is built to translate the generated landmarks into face images. During this stage, prior appearance information is extracted from the lower-half occluded target face and static reference images, which helps generate realistic and identity-preserving visual content. For effectively exploring the prior information of static reference images, we align static reference images with the target face's pose and expression based on motion fields. Moreover, auditory features are reused to guarantee that the generated face images are well synchronized with the audio. Extensive experiments demonstrate that our method can produce more realistic, lip-synced, and identity-preserving videos than existing person-generic talking face generation methods.","url_abs":"https://arxiv.org/abs/2305.08293v1","url_pdf":"https://arxiv.org/pdf/2305.08293v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"identity-preserving-talking-face-generation","repo_url":"https://github.com/Weizhi-Zhong/IP_LAP","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"face-generation","task_name":"Face Generation"},{"task_slug":"talking-face-generation","task_name":"Talking Face Generation"}],"methods":[{"method_slug":"align","method_name":"ALIGN"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2305.08293","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.08293"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Weizhi-Zhong/IP_LAP","reach":null}],"summary":{"ran":4,"unverified":2},"by_repo_kind":{"official":{"samples":6,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"1e34ba884cfc60b4","entry":"Conv1d","repo":"Weizhi-Zhong/IP_LAP","repo_kind":"official","path":"models/landmark_generator.py","file_url":"https://github.com/Weizhi-Zhong/IP_LAP/blob/HEAD/models/landmark_generator.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1e34ba884cfc60b4"}},{"code_sha256_prefix":"64c4ace973c8c4a6","entry":"Conv2d","repo":"Weizhi-Zhong/IP_LAP","repo_kind":"official","path":"models/landmark_generator.py","file_url":"https://github.com/Weizhi-Zhong/IP_LAP/blob/HEAD/models/landmark_generator.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"64c4ace973c8c4a6"}},{"code_sha256_prefix":"01b2c503a6c312f0","entry":"Fusion_transformer_encoder","repo":"Weizhi-Zhong/IP_LAP","repo_kind":"official","path":"models/landmark_generator.py","file_url":"https://github.com/Weizhi-Zhong/IP_LAP/blob/HEAD/models/landmark_generator.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"01b2c503a6c312f0"}},{"code_sha256_prefix":"a194859bc1a0f940","entry":"PositionalEmbedding","repo":"Weizhi-Zhong/IP_LAP","repo_kind":"official","path":"models/landmark_generator.py","file_url":"https://github.com/Weizhi-Zhong/IP_LAP/blob/HEAD/models/landmark_generator.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a194859bc1a0f940"}},{"code_sha256_prefix":"d6518309f929e38e","entry":"Landmark_generator","repo":"Weizhi-Zhong/IP_LAP","repo_kind":"official","path":"models/landmark_generator.py","file_url":"https://github.com/Weizhi-Zhong/IP_LAP/blob/HEAD/models/landmark_generator.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d6518309f929e38e"}},{"code_sha256_prefix":"4d77c95665872e53","entry":"weight_init","repo":"Weizhi-Zhong/IP_LAP","repo_kind":"official","path":"models/landmark_generator.py","file_url":"https://github.com/Weizhi-Zhong/IP_LAP/blob/HEAD/models/landmark_generator.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4d77c95665872e53"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}