{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/emotalk-speech-driven-emotional","title":"EmoTalk: Speech-Driven Emotional Disentanglement for 3D Face Animation","arxiv_id":"2303.11089","date":"2023-03-20","proceeding":"ICCV 2023 1","authors":["Ziqiao Peng","HaoYu Wu","Zhenbo Song","Hao Xu","Xiangyu Zhu","Jun He","Hongyan Liu","Zhaoxin Fan"],"abstract":"Speech-driven 3D face animation aims to generate realistic facial expressions that match the speech content and emotion. However, existing methods often neglect emotional facial expressions or fail to disentangle them from speech content. To address this issue, this paper proposes an end-to-end neural network to disentangle different emotions in speech so as to generate rich 3D facial expressions. Specifically, we introduce the emotion disentangling encoder (EDE) to disentangle the emotion and content in the speech by cross-reconstructed speech signals with different emotion labels. Then an emotion-guided feature fusion decoder is employed to generate a 3D talking face with enhanced emotion. The decoder is driven by the disentangled identity, emotional, and content embeddings so as to generate controllable personal and emotional styles. Finally, considering the scarcity of the 3D emotional talking face data, we resort to the supervision of facial blendshapes, which enables the reconstruction of plausible 3D faces from 2D emotional data, and contribute a large-scale 3D emotional talking face dataset (3D-ETF) to train the network. Our experiments and user studies demonstrate that our approach outperforms state-of-the-art methods and exhibits more diverse facial movements. We recommend watching the supplementary video: https://ziqiaopeng.github.io/emotalk","url_abs":"https://arxiv.org/abs/2303.11089v2","url_pdf":"https://arxiv.org/pdf/2303.11089v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"emotalk-speech-driven-emotional","repo_url":"https://github.com/psyai-net/EmoTalk_release","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"emotalk-speech-driven-emotional","repo_url":"https://github.com/ZiqiaoPeng/EmoTalk","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}}],"tasks":[{"task_slug":"3d-face-animation","task_name":"3D Face Animation"},{"task_slug":"decoder","task_name":"Decoder"},{"task_slug":"disentanglement","task_name":"Disentanglement"}],"methods":[{"method_slug":"fail","method_name":"fail"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2303.11089","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.11089"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ZiqiaoPeng/EmoTalk","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/psyai-net/EmoTalk_release","reach":null}],"summary":{"ran":1,"ran_violates":1,"ran_honours":1,"ran_fixture":1,"unverified":3},"by_repo_kind":{"official":{"samples":6,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":7,"samples":[{"code_sha256_prefix":"0592e2c94f31ace5","entry":"SpeechClassifierOutput","repo":"psyai-net/EmoTalk_release","repo_kind":"official","path":"model.py","file_url":"https://github.com/psyai-net/EmoTalk_release/blob/HEAD/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"0592e2c94f31ace5"}},{"code_sha256_prefix":"7ed01fed1fc1f58f","entry":"enc_dec_mask","repo":"psyai-net/EmoTalk_release","repo_kind":"official","path":"model.py","file_url":"https://github.com/psyai-net/EmoTalk_release/blob/HEAD/model.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"7ed01fed1fc1f58f"}},{"code_sha256_prefix":"f7b0318637b0548a","entry":"init_biased_mask","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"f7b0318637b0548a"}},{"code_sha256_prefix":"e45a37313dcbbaac","entry":"linear_interpolation","repo":"psyai-net/EmoTalk_release","repo_kind":"official","path":"model.py","file_url":"https://github.com/psyai-net/EmoTalk_release/blob/HEAD/model.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"e45a37313dcbbaac"}},{"code_sha256_prefix":"67df67021d9f046f","entry":"EmoTalk","repo":"psyai-net/EmoTalk_release","repo_kind":"official","path":"model.py","file_url":"https://github.com/psyai-net/EmoTalk_release/blob/HEAD/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"67df67021d9f046f"}},{"code_sha256_prefix":"b2ded1b916a526d9","entry":"Wav2Vec2ClassificationHead","repo":"psyai-net/EmoTalk_release","repo_kind":"official","path":"model.py","file_url":"https://github.com/psyai-net/EmoTalk_release/blob/HEAD/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"b2ded1b916a526d9"}},{"code_sha256_prefix":"bb65f17780694e75","entry":"Wav2Vec2ForSpeechClassification","repo":"psyai-net/EmoTalk_release","repo_kind":"official","path":"model.py","file_url":"https://github.com/psyai-net/EmoTalk_release/blob/HEAD/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"bb65f17780694e75"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}