{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/codified-audio-language-modeling-learns","title":"Codified audio language modeling learns useful representations for music information retrieval","arxiv_id":"2107.05677","date":"2021-07-12","proceeding":null,"authors":["Rodrigo Castellon","Chris Donahue","Percy Liang"],"abstract":"We demonstrate that language models pre-trained on codified (discretely-encoded) music audio learn representations that are useful for downstream MIR tasks. Specifically, we explore representations from Jukebox (Dhariwal et al. 2020): a music generation system containing a language model trained on codified audio from 1M songs. To determine if Jukebox's representations contain useful information for MIR, we use them as input features to train shallow models on several MIR tasks. Relative to representations from conventional MIR models which are pre-trained on tagging, we find that using representations from Jukebox as input features yields 30% stronger performance on average across four MIR tasks: tagging, genre classification, emotion recognition, and key detection. For key detection, we observe that representations from Jukebox are considerably stronger than those from models pre-trained on tagging, suggesting that pre-training via codified audio language modeling may address blind spots in conventional approaches. We interpret the strength of Jukebox's representations as evidence that modeling audio instead of tags provides richer representations for MIR.","url_abs":"https://arxiv.org/abs/2107.05677v1","url_pdf":"https://arxiv.org/pdf/2107.05677v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"codified-audio-language-modeling-learns","repo_url":"https://github.com/p-lambda/jukemir","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":null}],"tasks":[{"task_slug":"emotion-recognition","task_name":"Emotion Recognition"},{"task_slug":"genre-classification","task_name":"Genre classification"},{"task_slug":"information-retrieval","task_name":"Information Retrieval"},{"task_slug":"key-detection","task_name":"Key Detection"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"music-generation","task_name":"Music Generation"},{"task_slug":"music-genre-classification","task_name":"Music Genre Classification"},{"task_slug":"music-information-retrieval","task_name":"Music Information Retrieval"},{"task_slug":"music-tagging","task_name":"Music Tagging"},{"task_slug":"retrieval","task_name":"Retrieval"}],"methods":[{"method_slug":"convolution","method_name":"Convolution"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dilated-convolution","method_name":"Dilated Convolution"},{"method_slug":"jukebox","method_name":"Jukebox"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"vq-vae","method_name":"VQ-VAE"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/emotion-recognition-on-emomusic","task":"Emotion Recognition","dataset":"Emomusic","model":"Jukebox (Pre-training: CALM)","rank_in_archive_order":4,"of":5,"metrics":{"EmoA":"72.1","EmoV":"61.7"},"uses_additional_data":false},{"leaderboard":"/sota/emotion-recognition-on-emomusic","task":"Emotion Recognition","dataset":"Emomusic","model":"CLMR (Pre-training: contrastive)","rank_in_archive_order":5,"of":5,"metrics":{"EmoA":"67.8","EmoV":"45.8"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2107.05677","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2107.05677"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/p-lambda/jukemir","reach":null}],"summary":{"ran_draft_wrong":3},"by_repo_kind":{"official":{"samples":3,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"bdd32614114e574d","entry":"compute_checksum","repo":"p-lambda/jukemir","repo_kind":"official","path":"jukemir/utils.py","file_url":"https://github.com/p-lambda/jukemir/blob/HEAD/jukemir/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bdd32614114e574d"}},{"code_sha256_prefix":"56301668696d7843","entry":"encode_base64","repo":"p-lambda/jukemir","repo_kind":"official","path":"jukemir/utils.py","file_url":"https://github.com/p-lambda/jukemir/blob/HEAD/jukemir/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"56301668696d7843"}},{"code_sha256_prefix":"c894f8ae09cd8180","entry":"run_cmd_sync","repo":"p-lambda/jukemir","repo_kind":"official","path":"jukemir/utils.py","file_url":"https://github.com/p-lambda/jukemir/blob/HEAD/jukemir/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c894f8ae09cd8180"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}