{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/robots-learn-social-skills-end-to-end","title":"Robots Learn Social Skills: End-to-End Learning of Co-Speech Gesture Generation for Humanoid Robots","arxiv_id":"1810.12541","date":"2018-10-30","proceeding":null,"authors":["Youngwoo Yoon","Woo-Ri Ko","Minsu Jang","Jaeyeon Lee","Jaehong Kim","Geehyuk Lee"],"abstract":"Co-speech gestures enhance interaction experiences between humans as well as between humans and robots. Existing robots use rule-based speech-gesture association, but this requires human labor and prior knowledge of experts to be implemented. We present a learning-based co-speech gesture generation that is learned from 52 h of TED talks. The proposed end-to-end neural network model consists of an encoder for speech text understanding and a decoder to generate a sequence of gestures. The model successfully produces various gestures including iconic, metaphoric, deictic, and beat gestures. In a subjective evaluation, participants reported that the gestures were human-like and matched the speech content. We also demonstrate a co-speech gesture with a NAO robot working in real time.","url_abs":"http://arxiv.org/abs/1810.12541v1","url_pdf":"http://arxiv.org/pdf/1810.12541v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"links_only","authors_date_abstract":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license), from the Kaggle arXiv metadata snapshot of 2026-09-12"},"code_links":[{"paper_slug":"robots-learn-social-skills-end-to-end","repo_url":"https://github.com/PantoMatrix/BEAT","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"robots-learn-social-skills-end-to-end","repo_url":"https://github.com/pieterwolfert/co-speech-humanoids","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"robots-learn-social-skills-end-to-end","repo_url":"https://github.com/youngwoo-yoon/Co-Speech_Gesture_Generation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"robots-learn-social-skills-end-to-end","repo_url":"https://github.com/youngwoo-yoon/youtube-gesture-dataset","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"BSD-3-Clause"}},{"paper_slug":"robots-learn-social-skills-end-to-end","repo_url":"https://github.com/ehwa009/Co-Speech_Gesture_Generation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[],"methods":[],"datasets_introduced":[{"slug":"ted-gesture-dataset","name":"TED Gesture Dataset","full_name":""}],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/1810.12541","atlas_url":"https://app.syntology.ai/?focus=1810.12541","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1810.12541"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ehwa009/Co-Speech_Gesture_Generation","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/youngwoo-yoon/youtube-gesture-dataset","reach":{"status":"ok","spdx":"BSD-3-Clause"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/youngwoo-yoon/Co-Speech_Gesture_Generation","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/PantoMatrix/BEAT","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/pieterwolfert/co-speech-humanoids","reach":null}],"summary":{"ran_honours":1,"ran_draft_wrong":2,"ran_fixture":1,"unverified":12},"by_repo_kind":{"listed":{"samples":16,"ran":4,"repositories":3}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":2,"samples":[{"code_sha256_prefix":"c4246a111279f4bf","entry":"encodeinputstring","repo":"pieterwolfert/co-speech-humanoids","repo_kind":"listed","path":"inference.py","file_url":"https://github.com/pieterwolfert/co-speech-humanoids/blob/HEAD/inference.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c4246a111279f4bf"}},{"code_sha256_prefix":"da23fd901ddaf194","entry":"get_pad_mask","repo":"ehwa009/Co-Speech_Gesture_Generation","repo_kind":"listed","path":"transformer/models.py","file_url":"https://github.com/ehwa009/Co-Speech_Gesture_Generation/blob/HEAD/transformer/models.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"da23fd901ddaf194"}},{"code_sha256_prefix":"b84290f81844cc1c","entry":"get_subsequent_mask","repo":"ehwa009/Co-Speech_Gesture_Generation","repo_kind":"listed","path":"transformer/models.py","file_url":"https://github.com/ehwa009/Co-Speech_Gesture_Generation/blob/HEAD/transformer/models.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b84290f81844cc1c"}},{"code_sha256_prefix":"3b0218c9534b6ee7","entry":"unicode_to_ascii","repo":"youngwoo-yoon/youtube-gesture-dataset","repo_kind":"listed","path":"script/make_ted_dataset.py","file_url":"https://github.com/youngwoo-yoon/youtube-gesture-dataset/blob/HEAD/script/make_ted_dataset.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"3b0218c9534b6ee7"}},{"code_sha256_prefix":"76294a3b3ad8a25b","entry":"draw_skeleton_on_image","repo":"youngwoo-yoon/youtube-gesture-dataset","repo_kind":"listed","path":"script/data_utils.py","file_url":"https://github.com/youngwoo-yoon/youtube-gesture-dataset/blob/HEAD/script/data_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"76294a3b3ad8a25b"}},{"code_sha256_prefix":"b8fce4e5f8180782","entry":"get_closest_skeleton","repo":"youngwoo-yoon/youtube-gesture-dataset","repo_kind":"listed","path":"script/data_utils.py","file_url":"https://github.com/youngwoo-yoon/youtube-gesture-dataset/blob/HEAD/script/data_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"b8fce4e5f8180782"}},{"code_sha256_prefix":"ac8986943eac334d","entry":"get_vid_from_filename","repo":"youngwoo-yoon/youtube-gesture-dataset","repo_kind":"listed","path":"script/run_openpose.py","file_url":"https://github.com/youngwoo-yoon/youtube-gesture-dataset/blob/HEAD/script/run_openpose.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"ac8986943eac334d"}},{"code_sha256_prefix":"eedbb532217dcb7f","entry":"is_list_empty","repo":"youngwoo-yoon/youtube-gesture-dataset","repo_kind":"listed","path":"script/data_utils.py","file_url":"https://github.com/youngwoo-yoon/youtube-gesture-dataset/blob/HEAD/script/data_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"eedbb532217dcb7f"}},{"code_sha256_prefix":"47e5434a74faaa2f","entry":"normalize_skeleton","repo":"youngwoo-yoon/youtube-gesture-dataset","repo_kind":"listed","path":"script/motion_preprocessor.py","file_url":"https://github.com/youngwoo-yoon/youtube-gesture-dataset/blob/HEAD/script/motion_preprocessor.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"47e5434a74faaa2f"}},{"code_sha256_prefix":"61474285bf6c43cc","entry":"normalize_string","repo":"youngwoo-yoon/youtube-gesture-dataset","repo_kind":"listed","path":"script/make_ted_dataset.py","file_url":"https://github.com/youngwoo-yoon/youtube-gesture-dataset/blob/HEAD/script/make_ted_dataset.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"61474285bf6c43cc"}},{"code_sha256_prefix":"b7fe79023c1fb93b","entry":"read_sceneinfo","repo":"youngwoo-yoon/youtube-gesture-dataset","repo_kind":"listed","path":"script/run_clip_filtering.py","file_url":"https://github.com/youngwoo-yoon/youtube-gesture-dataset/blob/HEAD/script/run_clip_filtering.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"b7fe79023c1fb93b"}},{"code_sha256_prefix":"5b4fc0454f5233d4","entry":"read_skeleton_json","repo":"youngwoo-yoon/youtube-gesture-dataset","repo_kind":"listed","path":"script/run_openpose.py","file_url":"https://github.com/youngwoo-yoon/youtube-gesture-dataset/blob/HEAD/script/run_openpose.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"5b4fc0454f5233d4"}},{"code_sha256_prefix":"b4d548c56111c912","entry":"read_subtitle","repo":"youngwoo-yoon/youtube-gesture-dataset","repo_kind":"listed","path":"script/make_ted_dataset.py","file_url":"https://github.com/youngwoo-yoon/youtube-gesture-dataset/blob/HEAD/script/make_ted_dataset.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"b4d548c56111c912"}},{"code_sha256_prefix":"00c7dd2d57e0080f","entry":"run_filtering","repo":"youngwoo-yoon/youtube-gesture-dataset","repo_kind":"listed","path":"script/run_clip_filtering.py","file_url":"https://github.com/youngwoo-yoon/youtube-gesture-dataset/blob/HEAD/script/run_clip_filtering.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"00c7dd2d57e0080f"}},{"code_sha256_prefix":"8587006901a615dd","entry":"speech2gesture","repo":"pieterwolfert/co-speech-humanoids","repo_kind":"listed","path":"inference.py","file_url":"https://github.com/pieterwolfert/co-speech-humanoids/blob/HEAD/inference.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8587006901a615dd"}},{"code_sha256_prefix":"79171f8ed610e990","entry":"video_filter","repo":"youngwoo-yoon/youtube-gesture-dataset","repo_kind":"listed","path":"script/download_video.py","file_url":"https://github.com/youngwoo-yoon/youtube-gesture-dataset/blob/HEAD/script/download_video.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"79171f8ed610e990"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}