{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/unimumo-unified-text-music-and-motion","title":"UniMuMo: Unified Text, Music and Motion Generation","arxiv_id":"2410.04534","date":"2024-10-06","proceeding":null,"authors":["Han Yang","Kun Su","Yutong Zhang","Jiaben Chen","Kaizhi Qian","Gaowen Liu","Chuang Gan"],"abstract":"We introduce UniMuMo, a unified multimodal model capable of taking arbitrary text, music, and motion data as input conditions to generate outputs across all three modalities. To address the lack of time-synchronized data, we align unpaired music and motion data based on rhythmic patterns to leverage existing large-scale music-only and motion-only datasets. By converting music, motion, and text into token-based representation, our model bridges these modalities through a unified encoder-decoder transformer architecture. To support multiple generation tasks within a single framework, we introduce several architectural improvements. We propose encoding motion with a music codebook, mapping motion into the same feature space as music. We introduce a music-motion parallel generation scheme that unifies all music and motion generation tasks into a single transformer decoder architecture with a single training task of music-motion joint generation. Moreover, the model is designed by fine-tuning existing pre-trained single-modality models, significantly reducing computational demands. Extensive experiments demonstrate that UniMuMo achieves competitive results on all unidirectional generation benchmarks across music, motion, and text modalities. Quantitative results are available in the \\href{https://hanyangclarence.github.io/unimumo_demo/}{project page}.","url_abs":"https://arxiv.org/abs/2410.04534v1","url_pdf":"https://arxiv.org/pdf/2410.04534v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"unimumo-unified-text-music-and-motion","repo_url":"https://github.com/hanyangclarence/UniMuMo","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"decoder","task_name":"Decoder"},{"task_slug":"motion-generation","task_name":"Motion Generation"}],"methods":[{"method_slug":"align","method_name":"ALIGN"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2410.04534","atlas_url":"https://app.syntology.ai/?focus=2410.04534","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.04534"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/hanyangclarence/UniMuMo","reach":{"status":"ok"}}],"summary":{"ran":6,"ran_draft_wrong":1,"unverified":1},"by_repo_kind":{"official":{"samples":8,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":8,"samples":[{"code_sha256_prefix":"4bdc0cc9b4d0d928","entry":"angle_between","repo":"hanyangclarence/UniMuMo","repo_kind":"official","path":"unimumo/alignment/visual_beat.py","file_url":"https://github.com/hanyangclarence/UniMuMo/blob/HEAD/unimumo/alignment/visual_beat.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4bdc0cc9b4d0d928"}},{"code_sha256_prefix":"ed96da0641c14d4b","entry":"d_x","repo":"hanyangclarence/UniMuMo","repo_kind":"official","path":"unimumo/alignment/visual_beat.py","file_url":"https://github.com/hanyangclarence/UniMuMo/blob/HEAD/unimumo/alignment/visual_beat.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ed96da0641c14d4b"}},{"code_sha256_prefix":"221b2d116fdf1032","entry":"get_obj_from_str","repo":"hanyangclarence/UniMuMo","repo_kind":"official","path":"unimumo/util.py","file_url":"https://github.com/hanyangclarence/UniMuMo/blob/HEAD/unimumo/util.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"221b2d116fdf1032"}},{"code_sha256_prefix":"230615b9581d6e9d","entry":"instantiate_from_config","repo":"hanyangclarence/UniMuMo","repo_kind":"official","path":"unimumo/util.py","file_url":"https://github.com/hanyangclarence/UniMuMo/blob/HEAD/unimumo/util.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"230615b9581d6e9d"}},{"code_sha256_prefix":"158ff9eaa14b362d","entry":"interp","repo":"hanyangclarence/UniMuMo","repo_kind":"official","path":"unimumo/alignment/interpolation.py","file_url":"https://github.com/hanyangclarence/UniMuMo/blob/HEAD/unimumo/alignment/interpolation.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"158ff9eaa14b362d"}},{"code_sha256_prefix":"8eab2b6b52a23d15","entry":"interpolate","repo":"hanyangclarence/UniMuMo","repo_kind":"official","path":"unimumo/alignment/interpolation.py","file_url":"https://github.com/hanyangclarence/UniMuMo/blob/HEAD/unimumo/alignment/interpolation.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8eab2b6b52a23d15"}},{"code_sha256_prefix":"b504614ee071c480","entry":"unit_vector","repo":"hanyangclarence/UniMuMo","repo_kind":"official","path":"unimumo/alignment/visual_beat.py","file_url":"https://github.com/hanyangclarence/UniMuMo/blob/HEAD/unimumo/alignment/visual_beat.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b504614ee071c480"}},{"code_sha256_prefix":"becfa63019187994","entry":"load_model_from_config","repo":"hanyangclarence/UniMuMo","repo_kind":"official","path":"unimumo/util.py","file_url":"https://github.com/hanyangclarence/UniMuMo/blob/HEAD/unimumo/util.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"becfa63019187994"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}