{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/musicldm-enhancing-novelty-in-text-to-music","title":"MusicLDM: Enhancing Novelty in Text-to-Music Generation Using Beat-Synchronous Mixup Strategies","arxiv_id":"2308.01546","date":"2023-08-03","proceeding":null,"authors":["Ke Chen","Yusong Wu","Haohe Liu","Marianna Nezhurina","Taylor Berg-Kirkpatrick","Shlomo Dubnov"],"abstract":"Diffusion models have shown promising results in cross-modal generation tasks, including text-to-image and text-to-audio generation. However, generating music, as a special type of audio, presents unique challenges due to limited availability of music data and sensitive issues related to copyright and plagiarism. In this paper, to tackle these challenges, we first construct a state-of-the-art text-to-music model, MusicLDM, that adapts Stable Diffusion and AudioLDM architectures to the music domain. We achieve this by retraining the contrastive language-audio pretraining model (CLAP) and the Hifi-GAN vocoder, as components of MusicLDM, on a collection of music data samples. Then, to address the limitations of training data and to avoid plagiarism, we leverage a beat tracking model and propose two different mixup strategies for data augmentation: beat-synchronous audio mixup and beat-synchronous latent mixup, which recombine training audio directly or via a latent embeddings space, respectively. Such mixup strategies encourage the model to interpolate between musical training samples and generate new music within the convex hull of the training data, making the generated music more diverse while still staying faithful to the corresponding style. In addition to popular evaluation metrics, we design several new evaluation metrics based on CLAP score to demonstrate that our proposed MusicLDM and beat-synchronous mixup strategies improve both the quality and novelty of generated music, as well as the correspondence between input text and generated music.","url_abs":"https://arxiv.org/abs/2308.01546v1","url_pdf":"https://arxiv.org/pdf/2308.01546v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"musicldm-enhancing-novelty-in-text-to-music","repo_url":"https://github.com/retrocirce/musicldm","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}}],"tasks":[{"task_slug":"audio-generation","task_name":"Audio Generation"},{"task_slug":"beat-tracking","task_name":"Beat Tracking"},{"task_slug":"data-augmentation","task_name":"Data Augmentation"},{"task_slug":"music-generation","task_name":"Music Generation"},{"task_slug":"text-to-music-generation","task_name":"Text-to-Music Generation"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"},{"method_slug":"hifi-gan","method_name":"HiFi-GAN"},{"method_slug":"mixup","method_name":"Mixup"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2308.01546","atlas_url":"https://app.syntology.ai/?focus=2308.01546","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2308.01546"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/retrocirce/musicldm","reach":{"status":"ok","spdx":"NOASSERTION"}}],"summary":{"ran_violates":1,"ran_honours":2,"ran":5,"ran_draft_wrong":1,"unverified":5},"by_repo_kind":{"official":{"samples":14,"ran":9,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":14,"samples":[{"code_sha256_prefix":"4cb732f513d69dfd","entry":"disabled_train","repo":"retrocirce/musicldm","repo_kind":"official","path":"interface/src/latent_diffusion/models/musicldm.py","file_url":"https://github.com/retrocirce/musicldm/blob/HEAD/interface/src/latent_diffusion/models/musicldm.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4cb732f513d69dfd"}},{"code_sha256_prefix":"a26f85d7c72ef39a","entry":"get_padding","repo":"retrocirce/musicldm","repo_kind":"official","path":"interface/src/hifigan/models.py","file_url":"https://github.com/retrocirce/musicldm/blob/HEAD/interface/src/hifigan/models.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a26f85d7c72ef39a"}},{"code_sha256_prefix":"f1cba67eb6d6e4ad","entry":"get_param_num","repo":"retrocirce/musicldm","repo_kind":"official","path":"interface/src/utilities/model.py","file_url":"https://github.com/retrocirce/musicldm/blob/HEAD/interface/src/utilities/model.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f1cba67eb6d6e4ad"}},{"code_sha256_prefix":"2a3717d0b50d1f7d","entry":"get_restore_step","repo":"retrocirce/musicldm","repo_kind":"official","path":"interface/src/utilities/tools.py","file_url":"https://github.com/retrocirce/musicldm/blob/HEAD/interface/src/utilities/tools.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2a3717d0b50d1f7d"}},{"code_sha256_prefix":"b1368330cf0f5642","entry":"isimage","repo":"retrocirce/musicldm","repo_kind":"official","path":"interface/src/latent_diffusion/util.py","file_url":"https://github.com/retrocirce/musicldm/blob/HEAD/interface/src/latent_diffusion/util.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b1368330cf0f5642"}},{"code_sha256_prefix":"d72762b700feee6f","entry":"ismap","repo":"retrocirce/musicldm","repo_kind":"official","path":"interface/src/latent_diffusion/util.py","file_url":"https://github.com/retrocirce/musicldm/blob/HEAD/interface/src/latent_diffusion/util.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d72762b700feee6f"}},{"code_sha256_prefix":"af72484d5cfd7097","entry":"load_state_dict","repo":"retrocirce/musicldm","repo_kind":"official","path":"interface/src/clap/clap_module/factory.py","file_url":"https://github.com/retrocirce/musicldm/blob/HEAD/interface/src/clap/clap_module/factory.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"af72484d5cfd7097"}},{"code_sha256_prefix":"8a42cc3d7c42a043","entry":"read_json","repo":"retrocirce/musicldm","repo_kind":"official","path":"interface/src/utilities/tools.py","file_url":"https://github.com/retrocirce/musicldm/blob/HEAD/interface/src/utilities/tools.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8a42cc3d7c42a043"}},{"code_sha256_prefix":"d48d8354986e3b0e","entry":"uniform_on_device","repo":"retrocirce/musicldm","repo_kind":"official","path":"interface/src/latent_diffusion/models/musicldm.py","file_url":"https://github.com/retrocirce/musicldm/blob/HEAD/interface/src/latent_diffusion/models/musicldm.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d48d8354986e3b0e"}},{"code_sha256_prefix":"a5f2acab9d29a2f9","entry":"create_model","repo":"retrocirce/musicldm","repo_kind":"official","path":"interface/src/clap/clap_module/factory.py","file_url":"https://github.com/retrocirce/musicldm/blob/HEAD/interface/src/clap/clap_module/factory.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"a5f2acab9d29a2f9"}},{"code_sha256_prefix":"9d666bfa7fed3edc","entry":"create_model_and_transforms","repo":"retrocirce/musicldm","repo_kind":"official","path":"interface/src/clap/clap_module/factory.py","file_url":"https://github.com/retrocirce/musicldm/blob/HEAD/interface/src/clap/clap_module/factory.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"9d666bfa7fed3edc"}},{"code_sha256_prefix":"972736e790f5af88","entry":"get_available_checkpoint_keys","repo":"retrocirce/musicldm","repo_kind":"official","path":"interface/src/utilities/model.py","file_url":"https://github.com/retrocirce/musicldm/blob/HEAD/interface/src/utilities/model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"972736e790f5af88"}},{"code_sha256_prefix":"f9bd2e83191afad1","entry":"log_txt_as_img","repo":"retrocirce/musicldm","repo_kind":"official","path":"interface/src/latent_diffusion/util.py","file_url":"https://github.com/retrocirce/musicldm/blob/HEAD/interface/src/latent_diffusion/util.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f9bd2e83191afad1"}},{"code_sha256_prefix":"c8d9d38b144e401f","entry":"md5_hash","repo":"retrocirce/musicldm","repo_kind":"official","path":"interface/src/utilities/tools.py","file_url":"https://github.com/retrocirce/musicldm/blob/HEAD/interface/src/utilities/tools.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c8d9d38b144e401f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}