{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/natural-language-supervision-for-general","title":"Natural Language Supervision for General-Purpose Audio Representations","arxiv_id":"2309.05767","date":"2023-09-11","proceeding":null,"authors":["Benjamin Elizalde","Soham Deshmukh","Huaming Wang"],"abstract":"Audio-Language models jointly learn multimodal text and audio representations that enable Zero-Shot inference. Models rely on the encoders to create powerful representations of the input and generalize to multiple tasks ranging from sounds, music, and speech. Although models have achieved remarkable performance, there is still a performance gap with task-specific models. In this paper, we propose a Contrastive Language-Audio Pretraining model that is pretrained with a diverse collection of 4.6M audio-text pairs employing two innovative encoders for Zero-Shot inference. To learn audio representations, we trained an audio encoder on 22 audio tasks, instead of the standard training of sound event classification. To learn language representations, we trained an autoregressive decoder-only model instead of the standard encoder-only models. Then, the audio and language representations are brought into a joint multimodal space using Contrastive Learning. We used our encoders to improve the downstream performance by a margin. We extensively evaluated the generalization of our representations on 26 downstream tasks, the largest in the literature. Our model achieves state of the art results in several tasks leading the way towards general-purpose audio representations.","url_abs":"https://arxiv.org/abs/2309.05767v2","url_pdf":"https://arxiv.org/pdf/2309.05767v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"links_only","authors_date_abstract":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license), from the Kaggle arXiv metadata snapshot of 2026-09-12"},"code_links":[{"paper_slug":"natural-language-supervision-for-general","repo_url":"https://github.com/microsoft/clap","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2309.05767","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2309.05767"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/microsoft/clap","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":3,"ran_fixture":3,"unverified":2},"by_repo_kind":{"official":{"samples":8,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"5337e68cdb46397d","entry":"do_mixup","repo":"microsoft/clap","repo_kind":"official","path":"msclap/models/pytorch_utils.py","file_url":"https://github.com/microsoft/clap/blob/HEAD/msclap/models/pytorch_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5337e68cdb46397d"}},{"code_sha256_prefix":"3ac6b7d76e8e3584","entry":"drop_path","repo":"microsoft/clap","repo_kind":"official","path":"msclap/models/htsat.py","file_url":"https://github.com/microsoft/clap/blob/HEAD/msclap/models/htsat.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3ac6b7d76e8e3584"}},{"code_sha256_prefix":"bdcac6c19196f655","entry":"interpolate","repo":"microsoft/clap","repo_kind":"official","path":"msclap/models/pytorch_utils.py","file_url":"https://github.com/microsoft/clap/blob/HEAD/msclap/models/pytorch_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bdcac6c19196f655"}},{"code_sha256_prefix":"6e703419174005ee","entry":"move_data_to_device","repo":"microsoft/clap","repo_kind":"official","path":"msclap/models/pytorch_utils.py","file_url":"https://github.com/microsoft/clap/blob/HEAD/msclap/models/pytorch_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6e703419174005ee"}},{"code_sha256_prefix":"605878f6dcef72e9","entry":"trunc_normal_","repo":"microsoft/clap","repo_kind":"official","path":"msclap/models/htsat.py","file_url":"https://github.com/microsoft/clap/blob/HEAD/msclap/models/htsat.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"605878f6dcef72e9"}},{"code_sha256_prefix":"f9fd6241d935f07b","entry":"window_partition","repo":"microsoft/clap","repo_kind":"official","path":"msclap/models/htsat.py","file_url":"https://github.com/microsoft/clap/blob/HEAD/msclap/models/htsat.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f9fd6241d935f07b"}},{"code_sha256_prefix":"8e5c2405268a200e","entry":"get_clapcap","repo":"microsoft/clap","repo_kind":"official","path":"msclap/models/mapper.py","file_url":"https://github.com/microsoft/clap/blob/HEAD/msclap/models/mapper.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8e5c2405268a200e"}},{"code_sha256_prefix":"2ecc1e3196019a78","entry":"read_config_as_args","repo":"microsoft/clap","repo_kind":"official","path":"msclap/models/utils.py","file_url":"https://github.com/microsoft/clap/blob/HEAD/msclap/models/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2ecc1e3196019a78"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}