{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/audiolm-a-language-modeling-approach-to-audio","title":"AudioLM: a Language Modeling Approach to Audio Generation","arxiv_id":"2209.03143","date":"2022-09-07","proceeding":null,"authors":["Zalán Borsos","Raphaël Marinier","Damien Vincent","Eugene Kharitonov","Olivier Pietquin","Matt Sharifi","Dominik Roblek","Olivier Teboul","David Grangier","Marco Tagliasacchi","Neil Zeghidour"],"abstract":"We introduce AudioLM, a framework for high-quality audio generation with long-term consistency. AudioLM maps the input audio to a sequence of discrete tokens and casts audio generation as a language modeling task in this representation space. We show how existing audio tokenizers provide different trade-offs between reconstruction quality and long-term structure, and we propose a hybrid tokenization scheme to achieve both objectives. Namely, we leverage the discretized activations of a masked language model pre-trained on audio to capture long-term structure and the discrete codes produced by a neural audio codec to achieve high-quality synthesis. By training on large corpora of raw audio waveforms, AudioLM learns to generate natural and coherent continuations given short prompts. When trained on speech, and without any transcript or annotation, AudioLM generates syntactically and semantically plausible speech continuations while also maintaining speaker identity and prosody for unseen speakers. Furthermore, we demonstrate how our approach extends beyond speech by generating coherent piano music continuations, despite being trained without any symbolic representation of music.","url_abs":"https://arxiv.org/abs/2209.03143v2","url_pdf":"https://arxiv.org/pdf/2209.03143v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"audiolm-a-language-modeling-approach-to-audio","repo_url":"https://github.com/RoganInglis/AudioLM","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"audiolm-a-language-modeling-approach-to-audio","repo_url":"https://github.com/fixie-ai/ultravox","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"audiolm-a-language-modeling-approach-to-audio","repo_url":"https://github.com/plachtaa/vall-e-x","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"audiolm-a-language-modeling-approach-to-audio","repo_url":"https://github.com/serp-ai/bark-with-voice-clone","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"audiolm-a-language-modeling-approach-to-audio","repo_url":"https://github.com/suno-ai/bark","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"audiolm-a-language-modeling-approach-to-audio","repo_url":"https://github.com/lucidrains/audiolm-pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"audio-generation","task_name":"Audio Generation"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"speech-to-speech-translation","task_name":"Speech-to-Speech Translation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/on-zerospeech-2019-english","task":"","dataset":"ZeroSpeech 2019 English","model":"","rank_in_archive_order":1,"of":1,"metrics":{"":"2"},"uses_additional_data":false},{"leaderboard":"/sota/speech-to-speech-translation-on-fleurs-x-eng","task":"Speech-to-Speech Translation","dataset":"FLEURS X-eng","model":"AudioPaLM2","rank_in_archive_order":5,"of":7,"metrics":{"ASR-BLEU":"24.0"},"uses_additional_data":false}],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2209.03143","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2209.03143"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/serp-ai/bark-with-voice-clone","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/RoganInglis/AudioLM","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/suno-ai/bark","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/lucidrains/audiolm-pytorch","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/plachtaa/vall-e-x","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/fixie-ai/ultravox","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":2,"ran_violates":2,"unverified":11},"by_repo_kind":{"listed":{"samples":12,"ran":1,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":3,"samples":[{"code_sha256_prefix":"fe5dd5258046898c","entry":"always","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"fe5dd5258046898c"}},{"code_sha256_prefix":"60fff7c3c400d7ff","entry":"default","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"60fff7c3c400d7ff"}},{"code_sha256_prefix":"aa5486a3650902d8","entry":"exists","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"aa5486a3650902d8"}},{"code_sha256_prefix":"aebb727eed75bd10","entry":"get_pylogger","repo":"RoganInglis/AudioLM","repo_kind":"listed","path":"src/utils/pylogger.py","file_url":"https://github.com/RoganInglis/AudioLM/blob/HEAD/src/utils/pylogger.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"aebb727eed75bd10"}},{"code_sha256_prefix":"9f5e9393b5863f5c","entry":"apply_lora","repo":"fixie-ai/ultravox","repo_kind":"listed","path":"ultravox/model/ultravox_model.py","file_url":"https://github.com/fixie-ai/ultravox/blob/HEAD/ultravox/model/ultravox_model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9f5e9393b5863f5c"}},{"code_sha256_prefix":"bc6eef90dd03c9ab","entry":"download_file_from_wandb","repo":"fixie-ai/ultravox","repo_kind":"listed","path":"ultravox/model/wandb_utils.py","file_url":"https://github.com/fixie-ai/ultravox/blob/HEAD/ultravox/model/wandb_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bc6eef90dd03c9ab"}},{"code_sha256_prefix":"b913d51ad8556db7","entry":"download_hf_model","repo":"fixie-ai/ultravox","repo_kind":"listed","path":"ultravox/model/hf_hub_utils.py","file_url":"https://github.com/fixie-ai/ultravox/blob/HEAD/ultravox/model/hf_hub_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b913d51ad8556db7"}},{"code_sha256_prefix":"4db619fa023a6d7f","entry":"get_artifact","repo":"fixie-ai/ultravox","repo_kind":"listed","path":"ultravox/model/wandb_utils.py","file_url":"https://github.com/fixie-ai/ultravox/blob/HEAD/ultravox/model/wandb_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4db619fa023a6d7f"}},{"code_sha256_prefix":"31f39389a04a28ff","entry":"get_audio_token_id","repo":"fixie-ai/ultravox","repo_kind":"listed","path":"ultravox/model/ultravox_tokenizer.py","file_url":"https://github.com/fixie-ai/ultravox/blob/HEAD/ultravox/model/ultravox_tokenizer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"31f39389a04a28ff"}},{"code_sha256_prefix":"ac337726126bc790","entry":"get_checkpoint_files","repo":"fixie-ai/ultravox","repo_kind":"listed","path":"ultravox/model/ultravox_model.py","file_url":"https://github.com/fixie-ai/ultravox/blob/HEAD/ultravox/model/ultravox_model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ac337726126bc790"}},{"code_sha256_prefix":"f6763b312827f28c","entry":"get_hf_model_id","repo":"fixie-ai/ultravox","repo_kind":"listed","path":"ultravox/model/hf_hub_utils.py","file_url":"https://github.com/fixie-ai/ultravox/blob/HEAD/ultravox/model/hf_hub_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f6763b312827f28c"}},{"code_sha256_prefix":"5d515c356c24aefa","entry":"is_cache_empty","repo":"fixie-ai/ultravox","repo_kind":"listed","path":"ultravox/model/ultravox_model.py","file_url":"https://github.com/fixie-ai/ultravox/blob/HEAD/ultravox/model/ultravox_model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5d515c356c24aefa"}},{"code_sha256_prefix":"c8f279e78f17a2dc","entry":"is_hf_model","repo":"fixie-ai/ultravox","repo_kind":"listed","path":"ultravox/model/hf_hub_utils.py","file_url":"https://github.com/fixie-ai/ultravox/blob/HEAD/ultravox/model/hf_hub_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c8f279e78f17a2dc"}},{"code_sha256_prefix":"6071008854e0bbee","entry":"is_wandb_url","repo":"fixie-ai/ultravox","repo_kind":"listed","path":"ultravox/model/wandb_utils.py","file_url":"https://github.com/fixie-ai/ultravox/blob/HEAD/ultravox/model/wandb_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6071008854e0bbee"}},{"code_sha256_prefix":"c1454628160a1ce9","entry":"rename_state_dict_for_lora","repo":"fixie-ai/ultravox","repo_kind":"listed","path":"ultravox/training/model_types.py","file_url":"https://github.com/fixie-ai/ultravox/blob/HEAD/ultravox/training/model_types.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c1454628160a1ce9"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}