{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/simple-and-controllable-music-generation","title":"Simple and Controllable Music Generation","arxiv_id":"2306.05284","date":"2023-06-08","proceeding":"NeurIPS 2023 11","authors":["Jade Copet","Felix Kreuk","Itai Gat","Tal Remez","David Kant","Gabriel Synnaeve","Yossi Adi","Alexandre Défossez"],"abstract":"We tackle the task of conditional music generation. We introduce MusicGen, a single Language Model (LM) that operates over several streams of compressed discrete music representation, i.e., tokens. Unlike prior work, MusicGen is comprised of a single-stage transformer LM together with efficient token interleaving patterns, which eliminates the need for cascading several models, e.g., hierarchically or upsampling. Following this approach, we demonstrate how MusicGen can generate high-quality samples, both mono and stereo, while being conditioned on textual description or melodic features, allowing better controls over the generated output. We conduct extensive empirical evaluation, considering both automatic and human studies, showing the proposed approach is superior to the evaluated baselines on a standard text-to-music benchmark. Through ablation studies, we shed light over the importance of each of the components comprising MusicGen. Music samples, code, and models are available at https://github.com/facebookresearch/audiocraft","url_abs":"https://arxiv.org/abs/2306.05284v3","url_pdf":"https://arxiv.org/pdf/2306.05284v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"simple-and-controllable-music-generation","repo_url":"https://github.com/facebookresearch/audiocraft","is_official":0,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"simple-and-controllable-music-generation","repo_url":"https://github.com/atharva20038/music4all","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":{"status":"ok"}},{"paper_slug":"simple-and-controllable-music-generation","repo_url":"https://github.com/collabora/whisperspeech","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"simple-and-controllable-music-generation","repo_url":"https://github.com/theodorblackbird/lina-speech","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"simple-and-controllable-music-generation","repo_url":"https://github.com/whisperspeech/whisperspeech","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"simple-and-controllable-music-generation","repo_url":"https://github.com/pwc-1/Paper-5/tree/main/musicgen","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null},{"paper_slug":"simple-and-controllable-music-generation","repo_url":"https://github.com/pwc-1/Paper-5/tree/main/musicgen_melody","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null},{"paper_slug":"simple-and-controllable-music-generation","repo_url":"https://github.com/pwc-1/Paper-9/tree/main/4/text-to-music","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"music-generation","task_name":"Music Generation"},{"task_slug":"text-to-music-generation","task_name":"Text-to-Music Generation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/text-to-music-generation-on-musiccaps","task":"Text-to-Music Generation","dataset":"MusicCaps","model":"MusicGen w/o melody (1.5B)","rank_in_archive_order":10,"of":21,"metrics":{"FAD":"3.4","KL_passt":"1.23"},"uses_additional_data":false},{"leaderboard":"/sota/text-to-music-generation-on-musiccaps","task":"Text-to-Music Generation","dataset":"MusicCaps","model":"MusicGen w/o melody (3.3B)","rank_in_archive_order":13,"of":21,"metrics":{"FAD":"3.8","FD_openl3":"197.12","KL_passt":"1.31"},"uses_additional_data":false},{"leaderboard":"/sota/text-to-music-generation-on-musiccaps","task":"Text-to-Music Generation","dataset":"MusicCaps","model":"MusicGen w/ random melody (1.5B)","rank_in_archive_order":16,"of":21,"metrics":{"FAD":"5.0","KL_passt":"1.31"},"uses_additional_data":false}],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2306.05284","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.05284"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/facebookresearch/audiocraft","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/whisperspeech/whisperspeech","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/pwc-1/Paper-5/tree/main/musicgen_melody","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/collabora/whisperspeech","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/pwc-1/Paper-5/tree/main/musicgen","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/atharva20038/music4all","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/pwc-1/Paper-9/tree/main/4/text-to-music","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/theodorblackbird/lina-speech","reach":null}],"summary":{"ran":7,"ran_fixture":2,"ran_draft_wrong":2,"unverified":7},"by_repo_kind":{"named_in_paper":{"samples":3,"ran":3,"repositories":1},"listed":{"samples":15,"ran":8,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":7,"samples":[{"code_sha256_prefix":"d7c009ddcc0fa372","entry":"MultiEmbedding","repo":"theodorblackbird/lina-speech","repo_kind":"listed","path":"model/lina.py","file_url":"https://github.com/theodorblackbird/lina-speech/blob/HEAD/model/lina.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"d7c009ddcc0fa372"}},{"code_sha256_prefix":"545e144abdc0bc99","entry":"MulticlassAccuracy","repo":"theodorblackbird/lina-speech","repo_kind":"listed","path":"model/lina.py","file_url":"https://github.com/theodorblackbird/lina-speech/blob/HEAD/model/lina.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"545e144abdc0bc99"}},{"code_sha256_prefix":"b94d3dcb2a918be3","entry":"get_adv_criterion","repo":"facebookresearch/audiocraft","repo_kind":"named_in_paper","path":"audiocraft/adversarial/losses.py","file_url":"https://github.com/facebookresearch/audiocraft/blob/HEAD/audiocraft/adversarial/losses.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b94d3dcb2a918be3"}},{"code_sha256_prefix":"88b0022a7cb492a1","entry":"get_fake_criterion","repo":"facebookresearch/audiocraft","repo_kind":"named_in_paper","path":"audiocraft/adversarial/losses.py","file_url":"https://github.com/facebookresearch/audiocraft/blob/HEAD/audiocraft/adversarial/losses.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"88b0022a7cb492a1"}},{"code_sha256_prefix":"31e35a10d885d5a5","entry":"get_real_criterion","repo":"facebookresearch/audiocraft","repo_kind":"named_in_paper","path":"audiocraft/adversarial/losses.py","file_url":"https://github.com/facebookresearch/audiocraft/blob/HEAD/audiocraft/adversarial/losses.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"31e35a10d885d5a5"}},{"code_sha256_prefix":"8a8edc42e5e25cea","entry":"logits_to_probs","repo":"collabora/whisperspeech","repo_kind":"listed","path":"whisperspeech/inference.py","file_url":"https://github.com/collabora/whisperspeech/blob/HEAD/whisperspeech/inference.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8a8edc42e5e25cea"}},{"code_sha256_prefix":"0b5f90e8856ad7af","entry":"multinomial_sample_one_no_sync","repo":"collabora/whisperspeech","repo_kind":"listed","path":"whisperspeech/inference.py","file_url":"https://github.com/collabora/whisperspeech/blob/HEAD/whisperspeech/inference.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0b5f90e8856ad7af"}},{"code_sha256_prefix":"7286741cac3a2cab","entry":"sequence_mask","repo":"theodorblackbird/lina-speech","repo_kind":"listed","path":"model/lina.py","file_url":"https://github.com/theodorblackbird/lina-speech/blob/HEAD/model/lina.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"7286741cac3a2cab"}},{"code_sha256_prefix":"a314b3fcba001058","entry":"sinusoids","repo":"collabora/whisperspeech","repo_kind":"listed","path":"whisperspeech/modules.py","file_url":"https://github.com/collabora/whisperspeech/blob/HEAD/whisperspeech/modules.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a314b3fcba001058"}},{"code_sha256_prefix":"9c43cea8408f44fb","entry":"topk_sampling","repo":"theodorblackbird/lina-speech","repo_kind":"listed","path":"model/lina.py","file_url":"https://github.com/theodorblackbird/lina-speech/blob/HEAD/model/lina.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"9c43cea8408f44fb"}},{"code_sha256_prefix":"84caf63d220bcde1","entry":"undelay_rvq","repo":"theodorblackbird/lina-speech","repo_kind":"listed","path":"model/lina.py","file_url":"https://github.com/theodorblackbird/lina-speech/blob/HEAD/model/lina.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"84caf63d220bcde1"}},{"code_sha256_prefix":"c1ae840cb12e7690","entry":"AttentiveRNN","repo":"theodorblackbird/lina-speech","repo_kind":"listed","path":"model/lina.py","file_url":"https://github.com/theodorblackbird/lina-speech/blob/HEAD/model/lina.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"c1ae840cb12e7690"}},{"code_sha256_prefix":"b16dd7e91ff0ea1c","entry":"Lina","repo":"theodorblackbird/lina-speech","repo_kind":"listed","path":"model/lina.py","file_url":"https://github.com/theodorblackbird/lina-speech/blob/HEAD/model/lina.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"b16dd7e91ff0ea1c"}},{"code_sha256_prefix":"4cbe50f8df8999aa","entry":"calc_len","repo":"collabora/whisperspeech","repo_kind":"listed","path":"whisperspeech/extract_spk_emb.py","file_url":"https://github.com/collabora/whisperspeech/blob/HEAD/whisperspeech/extract_spk_emb.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4cbe50f8df8999aa"}},{"code_sha256_prefix":"e03ec9babe137321","entry":"load_model","repo":"collabora/whisperspeech","repo_kind":"listed","path":"whisperspeech/inference.py","file_url":"https://github.com/collabora/whisperspeech/blob/HEAD/whisperspeech/inference.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e03ec9babe137321"}},{"code_sha256_prefix":"97b1bc4db972b9ca","entry":"rope_rotate","repo":"collabora/whisperspeech","repo_kind":"listed","path":"whisperspeech/modules.py","file_url":"https://github.com/collabora/whisperspeech/blob/HEAD/whisperspeech/modules.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"97b1bc4db972b9ca"}},{"code_sha256_prefix":"72ac2b9f11df2bb5","entry":"rotate_half","repo":"collabora/whisperspeech","repo_kind":"listed","path":"whisperspeech/modules.py","file_url":"https://github.com/collabora/whisperspeech/blob/HEAD/whisperspeech/modules.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"72ac2b9f11df2bb5"}},{"code_sha256_prefix":"e67183981542ea8a","entry":"to_id","repo":"collabora/whisperspeech","repo_kind":"listed","path":"whisperspeech/languages.py","file_url":"https://github.com/collabora/whisperspeech/blob/HEAD/whisperspeech/languages.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e67183981542ea8a"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}