{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/bailando-3d-dance-generation-by-actor-critic","title":"Bailando: 3D Dance Generation by Actor-Critic GPT with Choreographic Memory","arxiv_id":"2203.13055","date":"2022-03-24","proceeding":"CVPR 2022 1","authors":["Li SiYao","Weijiang Yu","Tianpei Gu","Chunze Lin","Quan Wang","Chen Qian","Chen Change Loy","Ziwei Liu"],"abstract":"Driving 3D characters to dance following a piece of music is highly challenging due to the spatial constraints applied to poses by choreography norms. In addition, the generated dance sequence also needs to maintain temporal coherency with different music genres. To tackle these challenges, we propose a novel music-to-dance framework, Bailando, with two powerful components: 1) a choreographic memory that learns to summarize meaningful dancing units from 3D pose sequence to a quantized codebook, 2) an actor-critic Generative Pre-trained Transformer (GPT) that composes these units to a fluent dance coherent to the music. With the learned choreographic memory, dance generation is realized on the quantized units that meet high choreography standards, such that the generated dancing sequences are confined within the spatial constraints. To achieve synchronized alignment between diverse motion tempos and music beats, we introduce an actor-critic-based reinforcement learning scheme to the GPT with a newly-designed beat-align reward function. Extensive experiments on the standard benchmark demonstrate that our proposed framework achieves state-of-the-art performance both qualitatively and quantitatively. Notably, the learned choreographic memory is shown to discover human-interpretable dancing-style poses in an unsupervised manner.","url_abs":"https://arxiv.org/abs/2203.13055v2","url_pdf":"https://arxiv.org/pdf/2203.13055v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"bailando-3d-dance-generation-by-actor-critic","repo_url":"https://github.com/lisiyao21/bailando","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"motion-synthesis","task_name":"Motion Synthesis"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"cosine-annealing","method_name":"Cosine Annealing"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"discriminative-fine-tuning","method_name":"Discriminative Fine-Tuning"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt","method_name":"GPT"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-cosine-annealing","method_name":"Linear Warmup With Cosine Annealing"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"},{"method_slug":"weight-decay","method_name":"Weight Decay"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/motion-synthesis-on-aist","task":"Motion Synthesis","dataset":"AIST++","model":"Bailando","rank_in_archive_order":5,"of":12,"metrics":{"Beat alignment score":"0.233","FID":"28.16"},"uses_additional_data":false},{"leaderboard":"/sota/motion-synthesis-on-finedance","task":"Motion Synthesis","dataset":"FineDance","model":"Bailando","rank_in_archive_order":5,"of":8,"metrics":{"BAS":"0.2029","fid_k":"82.81"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2203.13055","atlas_url":"https://app.syntology.ai/?focus=2203.13055","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.13055"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/lisiyao21/bailando","reach":null}],"summary":{"ran":4,"unverified":1},"by_repo_kind":{"official":{"samples":5,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":5,"samples":[{"code_sha256_prefix":"7726bbe05a627078","entry":"Block","repo":"lisiyao21/bailando","repo_kind":"official","path":"models/cross_cond_gpt2_ac.py","file_url":"https://github.com/lisiyao21/bailando/blob/HEAD/models/cross_cond_gpt2_ac.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"7726bbe05a627078"}},{"code_sha256_prefix":"91a9c0928a4de128","entry":"CausalCrossConditionalSelfAttention","repo":"lisiyao21/bailando","repo_kind":"official","path":"models/cross_cond_gpt2_ac.py","file_url":"https://github.com/lisiyao21/bailando/blob/HEAD/models/cross_cond_gpt2_ac.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"91a9c0928a4de128"}},{"code_sha256_prefix":"5b09a79333f614e8","entry":"CrossCondGPTBase","repo":"lisiyao21/bailando","repo_kind":"official","path":"models/cross_cond_gpt2_ac.py","file_url":"https://github.com/lisiyao21/bailando/blob/HEAD/models/cross_cond_gpt2_ac.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"5b09a79333f614e8"}},{"code_sha256_prefix":"af8192e97c327c51","entry":"CrossCondGPTHead","repo":"lisiyao21/bailando","repo_kind":"official","path":"models/cross_cond_gpt2_ac.py","file_url":"https://github.com/lisiyao21/bailando/blob/HEAD/models/cross_cond_gpt2_ac.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"af8192e97c327c51"}},{"code_sha256_prefix":"6a9932a7b9f6f19d","entry":"CrossCondGPT2AC","repo":"lisiyao21/bailando","repo_kind":"official","path":"models/cross_cond_gpt2_ac.py","file_url":"https://github.com/lisiyao21/bailando/blob/HEAD/models/cross_cond_gpt2_ac.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"6a9932a7b9f6f19d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}