{"url":"/task/audio-generation","name":"Audio Generation","slug":"audio-generation","description_markdown":"Audio generation (synthesis) is the task of generating raw audio such as speech.\r\n\r\n<span style=\"color:grey; opacity: 0.6\">( Image credit: [MelNet](https://arxiv.org/pdf/1906.01083v1.pdf) )</span>","categories":[{"name":"Audio","url":"/area/audio"},{"name":"Music","url":"/area/music"},{"name":"Speech","url":"/area/speech"}],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":270,"papers_with_code":124,"benchmarks":3,"benchmark_tables_in_archive":3,"benchmark_tables_shown":3,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":11,"subtasks":4,"parent_tasks":0},"benchmarks":[{"leaderboard":"/sota/audio-generation-on-audiocaps","slug":"audio-generation-on-audiocaps","dataset":"AudioCaps","dataset_url":"/dataset/audiocaps","rows_in_archive":23,"metrics":["FD_openl3","FAD","FD","KL_passt","IS","CLAP_LAION","CLAP_MS"],"first_row_in_archive_order":{"model":"ETTA-FT-AC-100k","paper_title":"ETTA: Elucidating the Design Space of Text-to-Audio Models","paper_url":"/paper/etta-elucidating-the-design-space-of-text-to","paper_date":"2024-12-26","arxiv_id":"2412.19351","code_links":[{"title":"NVIDIA/elucidated-text-to-audio","url":"https://github.com/NVIDIA/elucidated-text-to-audio"}],"syntology":null}},{"leaderboard":"/sota/audio-generation-on-classical-music-5-seconds","slug":"audio-generation-on-classical-music-5-seconds","dataset":"Classical music, 5 seconds at 12 kHz","dataset_url":null,"rows_in_archive":2,"metrics":["Bits per byte"],"first_row_in_archive_order":{"model":"Sparse Transformer 152M (strided)","paper_title":"Generating Long Sequences with Sparse Transformers","paper_url":"/paper/190410509","paper_date":"2019-04-23","arxiv_id":"1904.10509","code_links":[{"title":"mistralai/mistral-src","url":"https://github.com/mistralai/mistral-src"},{"title":"openai/sparse_attention","url":"https://github.com/openai/sparse_attention"},{"title":"wilson1yan/VideoGPT","url":"https://github.com/wilson1yan/VideoGPT"},{"title":"ptillet/torch-blocksparse","url":"https://github.com/ptillet/torch-blocksparse"},{"title":"han-shi/SparseBERT","url":"https://github.com/han-shi/SparseBERT"},{"title":"jonahwinninghoff/Text-Summarization","url":"https://github.com/jonahwinninghoff/Text-Summarization"},{"title":"MindCode-4/code-11","url":"https://github.com/MindCode-4/code-11/tree/main/factorized-attention"}],"syntology":{"n":6,"n_ran":5,"n_unverified":1,"n_pointer_only":0}}},{"leaderboard":"/sota/audio-generation-on-symphony-music","slug":"audio-generation-on-symphony-music","dataset":"Symphony music","dataset_url":"/dataset/symphonynet","rows_in_archive":1,"metrics":[" Human listening average results"],"first_row_in_archive_order":{"model":"SymphonyNet","paper_title":"Symphony Generation with Permutation Invariant Language Model","paper_url":"/paper/symphony-generation-with-permutation","paper_date":"2022-05-10","arxiv_id":"2205.05448","code_links":[{"title":"symphonynet/SymphonyNet","url":"https://github.com/symphonynet/SymphonyNet"}],"syntology":{"n":2,"n_ran":2,"n_unverified":0,"n_pointer_only":0}}}],"datasets":[{"url":"/dataset/audiocaps","name":"AudioCaps","full_name":"","num_papers_in_archive":279},{"url":"/dataset/nsynth","name":"NSynth","full_name":"NSynth","num_papers_in_archive":138},{"url":"/dataset/maestro","name":"MAESTRO","full_name":"MAESTRO","num_papers_in_archive":118},{"url":"/dataset/movienet","name":"MovieNet","full_name":"MovieNet","num_papers_in_archive":54},{"url":"/dataset/symphonynet","name":"SymphonyNet","full_name":"SymphonyNet","num_papers_in_archive":10},{"url":"/dataset/nes-mdb","name":"NES-MDB","full_name":"Nintendo Entertainment System Music Database","num_papers_in_archive":7},{"url":"/dataset/adl-piano-midi","name":"ADL Piano MIDI","full_name":"","num_papers_in_archive":5},{"url":"/dataset/blizzard-challenge-2013","name":"Blizzard Challenge 2013","full_name":"Blizzard Challenge 2013 - English language tasks","num_papers_in_archive":5},{"url":"/dataset/dmelodies","name":"dMelodies","full_name":null,"num_papers_in_archive":4},{"url":"/dataset/audio-alpaca","name":"Audio-alpaca","full_name":"","num_papers_in_archive":2},{"url":"/dataset/medibeng","name":"MediBeng","full_name":"Synthetic Code-Switched Bengali-English Speech Conversations for Healthcare Applications","num_papers_in_archive":1}],"subtasks":[{"url":"/task/audio-super-resolution","name":"Audio Super-Resolution"},{"url":"/task/room-impulse-response-rir","name":"Room Impulse Response (RIR)"},{"url":"/task/video-to-sound-generation","name":"Video-to-Sound Generation"},{"url":"/task/voice-cloning","name":"Voice Cloning"}],"parent_tasks":[],"papers":{"order":"repositories listed in the archive (desc), then date (desc); the archive holds no stars","population":"papers tagged with this task that list at least one repository in the archive","shown":30,"of":124,"tagged_in_all":270,"items":[{"url":"/paper/wavenet-a-generative-model-for-raw-audio","title":"WaveNet: A Generative Model for Raw Audio","date":"2016-09-12","arxiv_id":"1609.03499","repositories_listed":62,"syntology":{"n":103,"n_ran":41,"n_unverified":62,"n_pointer_only":25}},{"url":"/paper/adversarial-audio-synthesis","title":"Adversarial Audio Synthesis","date":"2018-02-12","arxiv_id":"1802.04208","repositories_listed":22,"syntology":{"n":4,"n_ran":2,"n_unverified":2,"n_pointer_only":4}},{"url":"/paper/audiolm-a-language-modeling-approach-to-audio","title":"AudioLM: a Language Modeling Approach to Audio Generation","date":"2022-09-07","arxiv_id":"2209.03143","repositories_listed":6,"syntology":{"n":15,"n_ran":4,"n_unverified":11,"n_pointer_only":3}},{"url":"/paper/it-s-raw-audio-generation-with-state-space","title":"It's Raw! Audio Generation with State-Space Models","date":"2022-02-20","arxiv_id":"2202.09729","repositories_listed":6,"syntology":{"n":6,"n_ran":6,"n_unverified":0,"n_pointer_only":4}},{"url":"/paper/gansynth-adversarial-neural-audio-synthesis","title":"GANSynth: Adversarial Neural Audio Synthesis","date":"2019-02-23","arxiv_id":"1902.08710","repositories_listed":6,"syntology":null},{"url":"/paper/bigvgan-a-universal-neural-vocoder-with-large","title":"BigVGAN: A Universal Neural Vocoder with Large-Scale Training","date":"2022-06-09","arxiv_id":"2206.04658","repositories_listed":5,"syntology":{"n":17,"n_ran":8,"n_unverified":9,"n_pointer_only":1}},{"url":"/paper/melnet-a-generative-model-for-audio-in-the","title":"MelNet: A Generative Model for Audio in the Frequency Domain","date":"2019-06-04","arxiv_id":"1906.01083","repositories_listed":5,"syntology":{"n":4,"n_ran":4,"n_unverified":0,"n_pointer_only":4}},{"url":"/paper/high-fidelity-audio-compression-with-improved","title":"High-Fidelity Audio Compression with Improved RVQGAN","date":"2023-06-11","arxiv_id":"2306.06546","repositories_listed":4,"syntology":{"n":38,"n_ran":27,"n_unverified":11,"n_pointer_only":0}},{"url":"/paper/audioldm-text-to-audio-generation-with-latent","title":"AudioLDM: Text-to-Audio Generation with Latent Diffusion Models","date":"2023-01-29","arxiv_id":"2301.12503","repositories_listed":4,"syntology":{"n":21,"n_ran":8,"n_unverified":13,"n_pointer_only":10}},{"url":"/paper/audio-super-resolution-using-neural-networks","title":"Audio Super Resolution using Neural Networks","date":"2017-08-02","arxiv_id":"1708.00853","repositories_listed":4,"syntology":{"n":4,"n_ran":4,"n_unverified":0,"n_pointer_only":3}},{"url":"/paper/samplernn-an-unconditional-end-to-end-neural","title":"SampleRNN: An Unconditional End-to-End Neural Audio Generation Model","date":"2016-12-22","arxiv_id":"1612.07837","repositories_listed":4,"syntology":{"n":11,"n_ran":0,"n_unverified":11,"n_pointer_only":0}},{"url":"/paper/soundstorm-efficient-parallel-audio","title":"SoundStorm: Efficient Parallel Audio Generation","date":"2023-05-16","arxiv_id":"2305.09636","repositories_listed":3,"syntology":{"n":11,"n_ran":6,"n_unverified":5,"n_pointer_only":3}},{"url":"/paper/enhancing-suno-s-bark-text-to-speech-model","title":"Enhancing Suno's Bark Text-to-Speech Model: Addressing Limitations Through Meta's Encodec and Pre-Trained Hubert","date":"2023-04-18","arxiv_id":null,"repositories_listed":3,"syntology":null},{"url":"/paper/differentiable-time-frequency-scattering-in","title":"Differentiable Time-Frequency Scattering on GPU","date":"2022-04-18","arxiv_id":"2204.08269","repositories_listed":3,"syntology":null},{"url":"/paper/hifi-a-unified-framework-for-neural-vocoding","title":"HiFi++: a Unified Framework for Bandwidth Extension and Speech Enhancement","date":"2022-03-24","arxiv_id":"2203.13086","repositories_listed":3,"syntology":{"n":12,"n_ran":9,"n_unverified":3,"n_pointer_only":1}},{"url":"/paper/taming-visually-guided-sound-generation","title":"Taming Visually Guided Sound Generation","date":"2021-10-17","arxiv_id":"2110.08791","repositories_listed":3,"syntology":{"n":7,"n_ran":1,"n_unverified":6,"n_pointer_only":0}},{"url":"/paper/ddsp-differentiable-digital-signal-processing-1","title":"DDSP: Differentiable Digital Signal Processing","date":"2020-01-14","arxiv_id":"2001.04643","repositories_listed":3,"syntology":{"n":5,"n_ran":0,"n_unverified":5,"n_pointer_only":5}},{"url":"/paper/190600794","title":"Blow: a single-scale hyperconditioned flow for non-parallel raw-audio voice conversion","date":"2019-06-03","arxiv_id":"1906.00794","repositories_listed":3,"syntology":{"n":5,"n_ran":1,"n_unverified":4,"n_pointer_only":0}},{"url":"/paper/assisted-sound-sample-generation-with-musical","title":"Assisted Sound Sample Generation with Musical Conditioning in Adversarial Auto-Encoders","date":"2019-04-12","arxiv_id":"1904.06215","repositories_listed":3,"syntology":null},{"url":"/paper/latent-swap-joint-diffusion-for-long-form","title":"Latent Swap Joint Diffusion for 2D Long-Form Latent Generation","date":"2025-02-07","arxiv_id":"2502.05130","repositories_listed":2,"syntology":null},{"url":"/paper/audio-deepfake-detection-with-self-supervised-1","title":"Audio Deepfake Detection with Self-Supervised XLS-R and SLS Classifier","date":"2024-10-28","arxiv_id":null,"repositories_listed":2,"syntology":null},{"url":"/paper/movie-gen-a-cast-of-media-foundation-models","title":"Movie Gen: A Cast of Media Foundation Models","date":"2024-10-17","arxiv_id":"2410.13720","repositories_listed":2,"syntology":null},{"url":"/paper/espnet-codec-comprehensive-training-and","title":"ESPnet-Codec: Comprehensive Training and Evaluation of Neural Codecs for Audio, Music, and Speech","date":"2024-09-24","arxiv_id":"2409.15897","repositories_listed":2,"syntology":null},{"url":"/paper/audiolcm-text-to-audio-generation-with-latent","title":"AudioLCM: Text-to-Audio Generation with Latent Consistency Models","date":"2024-06-01","arxiv_id":"2406.00356","repositories_listed":2,"syntology":null},{"url":"/paper/fast-timing-conditioned-latent-audio","title":"Fast Timing-Conditioned Latent Audio Diffusion","date":"2024-02-07","arxiv_id":"2402.04825","repositories_listed":2,"syntology":{"n":4,"n_ran":2,"n_unverified":2,"n_pointer_only":0}},{"url":"/paper/invisible-watermarking-for-audio-generation","title":"Invisible Watermarking for Audio Generation Diffusion Models","date":"2023-09-22","arxiv_id":"2309.13166","repositories_listed":2,"syntology":null},{"url":"/paper/audioldm-2-learning-holistic-audio-generation","title":"AudioLDM 2: Learning Holistic Audio Generation with Self-supervised Pretraining","date":"2023-08-10","arxiv_id":"2308.05734","repositories_listed":2,"syntology":{"n":27,"n_ran":12,"n_unverified":15,"n_pointer_only":13}},{"url":"/paper/musecoco-generating-symbolic-music-from-text","title":"MuseCoco: Generating Symbolic Music from Text","date":"2023-05-31","arxiv_id":"2306.00110","repositories_listed":2,"syntology":null},{"url":"/paper/any-to-any-generation-via-composable","title":"Any-to-Any Generation via Composable Diffusion","date":"2023-05-19","arxiv_id":"2305.11846","repositories_listed":2,"syntology":{"n":13,"n_ran":2,"n_unverified":11,"n_pointer_only":0}},{"url":"/paper/multi-singer-fast-multi-singer-singing-voice-1","title":"Multi-Singer: Fast Multi-Singer Singing Voice Vocoder With A Large-Scale Corpus","date":"2021-12-20","arxiv_id":"2112.10358","repositories_listed":2,"syntology":{"n":2,"n_ran":0,"n_unverified":2,"n_pointer_only":0}}],"syntology_records":19,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-09-24T18:15:14+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"}},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"}}