{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/audioldm-text-to-audio-generation-with-latent","title":"AudioLDM: Text-to-Audio Generation with Latent Diffusion Models","arxiv_id":"2301.12503","date":"2023-01-29","proceeding":null,"authors":["Haohe Liu","Zehua Chen","Yi Yuan","Xinhao Mei","Xubo Liu","Danilo Mandic","Wenwu Wang","Mark D. Plumbley"],"abstract":"Text-to-audio (TTA) system has recently gained attention for its ability to synthesize general audio based on text descriptions. However, previous studies in TTA have limited generation quality with high computational costs. In this study, we propose AudioLDM, a TTA system that is built on a latent space to learn the continuous audio representations from contrastive language-audio pretraining (CLAP) latents. The pretrained CLAP models enable us to train LDMs with audio embedding while providing text embedding as a condition during sampling. By learning the latent representations of audio signals and their compositions without modeling the cross-modal relationship, AudioLDM is advantageous in both generation quality and computational efficiency. Trained on AudioCaps with a single GPU, AudioLDM achieves state-of-the-art TTA performance measured by both objective and subjective metrics (e.g., frechet distance). Moreover, AudioLDM is the first TTA system that enables various text-guided audio manipulations (e.g., style transfer) in a zero-shot fashion. Our implementation and demos are available at https://audioldm.github.io.","url_abs":"https://arxiv.org/abs/2301.12503v3","url_pdf":"https://arxiv.org/pdf/2301.12503v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"audioldm-text-to-audio-generation-with-latent","repo_url":"https://github.com/haoheliu/AudioLDM","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"audioldm-text-to-audio-generation-with-latent","repo_url":"https://github.com/haoheliu/audioldm_eval","is_official":0,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"audioldm-text-to-audio-generation-with-latent","repo_url":"https://github.com/haoheliu/audioldm-training-finetuning","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"audioldm-text-to-audio-generation-with-latent","repo_url":"https://github.com/huggingface/diffusers","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"jax","reach":null}],"tasks":[{"task_slug":"audio-generation","task_name":"Audio Generation"},{"task_slug":null,"task_name":"AudioCaps"},{"task_slug":"computational-efficiency","task_name":"Computational Efficiency"},{"task_slug":null,"task_name":"GPU"},{"task_slug":"style-transfer","task_name":"Style Transfer"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/audio-generation-on-audiocaps","task":"Audio Generation","dataset":"AudioCaps","model":"AudioLDM-L-Full","rank_in_archive_order":19,"of":23,"metrics":{"FAD":"1.96","FD":"23.31"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2301.12503","atlas_url":"https://app.syntology.ai/?focus=2301.12503","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2301.12503"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/huggingface/diffusers","reach":null},{"provenance":"deterministic:regex_extraction","url":"https://github.com/haoheliu/audioldm_","reach":{"status":"gone","observed_at":"2026-09-17","how":"tree_404+repo_404"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/haoheliu/audioldm_eval","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/haoheliu/audioldm-training-finetuning","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/haoheliu/AudioLDM","reach":{"status":"ok","spdx":"NOASSERTION"}}],"summary":{"ran":9,"ran_violates":1,"ran_draft_wrong":2,"ran_fixture":2,"ran_honours":1,"unverified":6},"by_repo_kind":{"official":{"samples":9,"ran":5,"repositories":1},"named_in_paper":{"samples":11,"ran":9,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":10,"samples":[{"code_sha256_prefix":"2cda99b6def85873","entry":"DiagonalGaussianDistribution","repo":"haoheliu/AudioLDM","repo_kind":"official","path":"audioldm/ldm.py","file_url":"https://github.com/haoheliu/AudioLDM/blob/HEAD/audioldm/ldm.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"2cda99b6def85873"}},{"code_sha256_prefix":"97d45ba74474fdfc","entry":"clip_bce","repo":"haoheliu/audioldm_eval","repo_kind":"named_in_paper","path":"audioldm_eval/feature_extractors/panns/losses.py","file_url":"https://github.com/haoheliu/audioldm_eval/blob/HEAD/audioldm_eval/feature_extractors/panns/losses.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"97d45ba74474fdfc"}},{"code_sha256_prefix":"4cb732f513d69dfd","entry":"disabled_train","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"4cb732f513d69dfd"}},{"code_sha256_prefix":"8971042e12016a60","entry":"extract_into_tensor","repo":"haoheliu/AudioLDM","repo_kind":"official","path":"audioldm/ldm.py","file_url":"https://github.com/haoheliu/AudioLDM/blob/HEAD/audioldm/ldm.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"8971042e12016a60"}},{"code_sha256_prefix":"e424ea90cdf3fd97","entry":"get_loss_func","repo":"haoheliu/audioldm_eval","repo_kind":"named_in_paper","path":"audioldm_eval/feature_extractors/panns/losses.py","file_url":"https://github.com/haoheliu/audioldm_eval/blob/HEAD/audioldm_eval/feature_extractors/panns/losses.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e424ea90cdf3fd97"}},{"code_sha256_prefix":"af0a67f05ca8144d","entry":"get_mel_from_wav","repo":"haoheliu/audioldm_eval","repo_kind":"named_in_paper","path":"audioldm_eval/audio/tools.py","file_url":"https://github.com/haoheliu/audioldm_eval/blob/HEAD/audioldm_eval/audio/tools.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"af0a67f05ca8144d"}},{"code_sha256_prefix":"9f9ec26d4cdfdf7d","entry":"griffin_lim","repo":"haoheliu/audioldm_eval","repo_kind":"named_in_paper","path":"audioldm_eval/audio/audio_processing.py","file_url":"https://github.com/haoheliu/audioldm_eval/blob/HEAD/audioldm_eval/audio/audio_processing.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9f9ec26d4cdfdf7d"}},{"code_sha256_prefix":"21e50186ab33dd24","entry":"interpolate_bilinear_2d_like_tensorflow1x","repo":"haoheliu/audioldm_eval","repo_kind":"named_in_paper","path":"audioldm_eval/feature_extractors/inception3.py","file_url":"https://github.com/haoheliu/audioldm_eval/blob/HEAD/audioldm_eval/feature_extractors/inception3.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"21e50186ab33dd24"}},{"code_sha256_prefix":"777e661635ced528","entry":"load_json","repo":"haoheliu/audioldm_eval","repo_kind":"named_in_paper","path":"audioldm_eval/audio/tools.py","file_url":"https://github.com/haoheliu/audioldm_eval/blob/HEAD/audioldm_eval/audio/tools.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"777e661635ced528"}},{"code_sha256_prefix":"ffc354ebaecde324","entry":"load_module2model","repo":"haoheliu/audioldm_eval","repo_kind":"named_in_paper","path":"audioldm_eval/feature_extractors/melception_audioset.py","file_url":"https://github.com/haoheliu/audioldm_eval/blob/HEAD/audioldm_eval/feature_extractors/melception_audioset.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ffc354ebaecde324"}},{"code_sha256_prefix":"78222542af18c6c5","entry":"load_pickle","repo":"haoheliu/audioldm_eval","repo_kind":"named_in_paper","path":"audioldm_eval/audio/tools.py","file_url":"https://github.com/haoheliu/audioldm_eval/blob/HEAD/audioldm_eval/audio/tools.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"78222542af18c6c5"}},{"code_sha256_prefix":"3ee640131c9d4362","entry":"make_ddim_sampling_parameters","repo":"haoheliu/AudioLDM","repo_kind":"official","path":"audioldm/ldm.py","file_url":"https://github.com/haoheliu/AudioLDM/blob/HEAD/audioldm/ldm.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"3ee640131c9d4362"}},{"code_sha256_prefix":"0ea4e960ea54514c","entry":"make_ddim_timesteps","repo":"haoheliu/AudioLDM","repo_kind":"official","path":"audioldm/ldm.py","file_url":"https://github.com/haoheliu/AudioLDM/blob/HEAD/audioldm/ldm.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"0ea4e960ea54514c"}},{"code_sha256_prefix":"2d7200ac60d9a4bd","entry":"noise_like","repo":"haoheliu/AudioLDM","repo_kind":"official","path":"audioldm/ldm.py","file_url":"https://github.com/haoheliu/AudioLDM/blob/HEAD/audioldm/ldm.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"2d7200ac60d9a4bd"}},{"code_sha256_prefix":"e0e3ae45a19356fa","entry":"pad_short_audio","repo":"haoheliu/audioldm_eval","repo_kind":"named_in_paper","path":"audioldm_eval/datasets/load_mel.py","file_url":"https://github.com/haoheliu/audioldm_eval/blob/HEAD/audioldm_eval/datasets/load_mel.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e0e3ae45a19356fa"}},{"code_sha256_prefix":"332deacf272549e4","entry":"DDIMSampler","repo":"haoheliu/AudioLDM","repo_kind":"official","path":"audioldm/ldm.py","file_url":"https://github.com/haoheliu/AudioLDM/blob/HEAD/audioldm/ldm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"332deacf272549e4"}},{"code_sha256_prefix":"b7521b6c520cc691","entry":"DDPM","repo":"haoheliu/AudioLDM","repo_kind":"official","path":"audioldm/ldm.py","file_url":"https://github.com/haoheliu/AudioLDM/blob/HEAD/audioldm/ldm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"b7521b6c520cc691"}},{"code_sha256_prefix":"bb9a12b66a9694f1","entry":"DiffusionWrapper","repo":"haoheliu/AudioLDM","repo_kind":"official","path":"audioldm/ldm.py","file_url":"https://github.com/haoheliu/AudioLDM/blob/HEAD/audioldm/ldm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"bb9a12b66a9694f1"}},{"code_sha256_prefix":"1d0e58f8ba043419","entry":"LatentDiffusion","repo":"haoheliu/AudioLDM","repo_kind":"official","path":"audioldm/ldm.py","file_url":"https://github.com/haoheliu/AudioLDM/blob/HEAD/audioldm/ldm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"1d0e58f8ba043419"}},{"code_sha256_prefix":"0b80ec09e3ba47ae","entry":"dynamic_range_compression","repo":"haoheliu/audioldm_eval","repo_kind":"named_in_paper","path":"audioldm_eval/audio/audio_processing.py","file_url":"https://github.com/haoheliu/audioldm_eval/blob/HEAD/audioldm_eval/audio/audio_processing.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0b80ec09e3ba47ae"}},{"code_sha256_prefix":"98e449b3a6b45350","entry":"load_npy_data","repo":"haoheliu/audioldm_eval","repo_kind":"named_in_paper","path":"audioldm_eval/datasets/load_mel.py","file_url":"https://github.com/haoheliu/audioldm_eval/blob/HEAD/audioldm_eval/datasets/load_mel.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"98e449b3a6b45350"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}