{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/high-fidelity-relightable-monocular-portrait","title":"High-Fidelity Relightable Monocular Portrait Animation with Lighting-Controllable Video Diffusion Model","arxiv_id":"2502.19894","date":"2025-02-27","proceeding":"CVPR 2025 1","authors":["Mingtao Guo","Guanyu Xing","Yanli Liu"],"abstract":"Relightable portrait animation aims to animate a static reference portrait to match the head movements and expressions of a driving video while adapting to user-specified or reference lighting conditions. Existing portrait animation methods fail to achieve relightable portraits because they do not separate and manipulate intrinsic (identity and appearance) and extrinsic (pose and lighting) features. In this paper, we present a Lighting Controllable Video Diffusion model (LCVD) for high-fidelity, relightable portrait animation. We address this limitation by distinguishing these feature types through dedicated subspaces within the feature space of a pre-trained image-to-video diffusion model. Specifically, we employ the 3D mesh, pose, and lighting-rendered shading hints of the portrait to represent the extrinsic attributes, while the reference represents the intrinsic attributes. In the training phase, we employ a reference adapter to map the reference into the intrinsic feature subspace and a shading adapter to map the shading hints into the extrinsic feature subspace. By merging features from these subspaces, the model achieves nuanced control over lighting, pose, and expression in generated animations. Extensive evaluations show that LCVD outperforms state-of-the-art methods in lighting realism, image quality, and video consistency, setting a new benchmark in relightable portrait animation.","url_abs":"https://arxiv.org/abs/2502.19894v1","url_pdf":"https://arxiv.org/pdf/2502.19894v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"high-fidelity-relightable-monocular-portrait","repo_url":"https://github.com/MingtaoGuo/relightable-portrait-animation","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"portrait-animation","task_name":"Portrait Animation"}],"methods":[{"method_slug":"adapter","method_name":"Adapter"},{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2502.19894","atlas_url":"https://app.syntology.ai/?focus=2502.19894","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2502.19894"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/MingtaoGuo/Relightable-Portrait-Animation","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":6,"ran_honours":1},"by_repo_kind":{"official":{"samples":7,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"274890e1b13dfed8","entry":"center_crop_cv2","repo":"MingtaoGuo/Relightable-Portrait-Animation","repo_kind":"official","path":"src/facepose/img_utils.py","file_url":"https://github.com/MingtaoGuo/Relightable-Portrait-Animation/blob/HEAD/src/facepose/img_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"274890e1b13dfed8"}},{"code_sha256_prefix":"3fbf6112079081f4","entry":"create_perspective_matrix","repo":"MingtaoGuo/Relightable-Portrait-Animation","repo_kind":"official","path":"src/facepose/motion_utils.py","file_url":"https://github.com/MingtaoGuo/Relightable-Portrait-Animation/blob/HEAD/src/facepose/motion_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3fbf6112079081f4"}},{"code_sha256_prefix":"03eba32dada60b1e","entry":"cv2_to_pil","repo":"MingtaoGuo/Relightable-Portrait-Animation","repo_kind":"official","path":"src/facepose/img_utils.py","file_url":"https://github.com/MingtaoGuo/Relightable-Portrait-Animation/blob/HEAD/src/facepose/img_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"03eba32dada60b1e"}},{"code_sha256_prefix":"89e4412c100b0f9b","entry":"normPRED","repo":"MingtaoGuo/Relightable-Portrait-Animation","repo_kind":"official","path":"src/facematting/u2net_matting.py","file_url":"https://github.com/MingtaoGuo/Relightable-Portrait-Animation/blob/HEAD/src/facematting/u2net_matting.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"89e4412c100b0f9b"}},{"code_sha256_prefix":"ecf514cc31b3a71d","entry":"pil_to_cv2","repo":"MingtaoGuo/Relightable-Portrait-Animation","repo_kind":"official","path":"src/facepose/img_utils.py","file_url":"https://github.com/MingtaoGuo/Relightable-Portrait-Animation/blob/HEAD/src/facepose/img_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ecf514cc31b3a71d"}},{"code_sha256_prefix":"98a46adba2cbcf92","entry":"project_points_with_trans","repo":"MingtaoGuo/Relightable-Portrait-Animation","repo_kind":"official","path":"src/facepose/motion_utils.py","file_url":"https://github.com/MingtaoGuo/Relightable-Portrait-Animation/blob/HEAD/src/facepose/motion_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"98a46adba2cbcf92"}},{"code_sha256_prefix":"292807b206887a39","entry":"project_vertices_from_ref2tgt","repo":"MingtaoGuo/Relightable-Portrait-Animation","repo_kind":"official","path":"src/facepose/motion_utils.py","file_url":"https://github.com/MingtaoGuo/Relightable-Portrait-Animation/blob/HEAD/src/facepose/motion_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"292807b206887a39"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}