{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/makeittalk-speaker-aware-talking-head","title":"MakeItTalk: Speaker-Aware Talking-Head Animation","arxiv_id":"2004.12992","date":"2020-04-27","proceeding":null,"authors":["Yang Zhou","Xintong Han","Eli Shechtman","Jose Echevarria","Evangelos Kalogerakis","DIngzeyu Li"],"abstract":"We present a method that generates expressive talking heads from a single facial image with audio as the only input. In contrast to previous approaches that attempt to learn direct mappings from audio to raw pixels or points for creating talking faces, our method first disentangles the content and speaker information in the input audio signal. The audio content robustly controls the motion of lips and nearby facial regions, while the speaker information determines the specifics of facial expressions and the rest of the talking head dynamics. Another key component of our method is the prediction of facial landmarks reflecting speaker-aware dynamics. Based on this intermediate representation, our method is able to synthesize photorealistic videos of entire talking heads with full range of motion and also animate artistic paintings, sketches, 2D cartoon characters, Japanese mangas, stylized caricatures in a single unified framework. We present extensive quantitative and qualitative evaluation of our method, in addition to user studies, demonstrating generated talking heads of significantly higher quality compared to prior state-of-the-art.","url_abs":"https://arxiv.org/abs/2004.12992v3","url_pdf":"https://arxiv.org/pdf/2004.12992v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"makeittalk-speaker-aware-talking-head","repo_url":"https://github.com/adobe-research/MakeItTalk","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"makeittalk-speaker-aware-talking-head","repo_url":"https://github.com/livingbio/MakeItTalk","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"makeittalk-speaker-aware-talking-head","repo_url":"https://github.com/yzhou359/MakeItTalk","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}}],"tasks":[{"task_slug":"talking-face-generation","task_name":"Talking Face Generation"},{"task_slug":"talking-head-generation","task_name":"Talking Head Generation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2004.12992","atlas_url":"https://app.syntology.ai/?focus=2004.12992","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.12992"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yzhou359/MakeItTalk","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/livingbio/MakeItTalk","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/adobe-research/MakeItTalk","reach":{"status":"ok","spdx":"NOASSERTION"}}],"summary":{"ran_honours":1,"ran_draft_wrong":1},"by_repo_kind":{"listed":{"samples":1,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":2,"samples":[{"code_sha256_prefix":"608920c78f1ae5e9","entry":"get_n_params","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"608920c78f1ae5e9"}},{"code_sha256_prefix":"9639b933989145e0","entry":"smooth","repo":"livingbio/MakeItTalk","repo_kind":"listed","path":"src/util/utils.py","file_url":"https://github.com/livingbio/MakeItTalk/blob/HEAD/src/util/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9639b933989145e0"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}