{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/enclap-combining-neural-audio-codec-and-audio","title":"EnCLAP: Combining Neural Audio Codec and Audio-Text Joint Embedding for Automated Audio Captioning","arxiv_id":"2401.17690","date":"2024-01-31","proceeding":null,"authors":["Jaeyeon Kim","JaeYoon Jung","Jinjoo Lee","Sang Hoon Woo"],"abstract":"We propose EnCLAP, a novel framework for automated audio captioning. EnCLAP employs two acoustic representation models, EnCodec and CLAP, along with a pretrained language model, BART. We also introduce a new training objective called masked codec modeling that improves acoustic awareness of the pretrained language model. Experimental results on AudioCaps and Clotho demonstrate that our model surpasses the performance of baseline models. Source code will be available at https://github.com/jaeyeonkim99/EnCLAP . An online demo is available at https://huggingface.co/spaces/enclap-team/enclap .","url_abs":"https://arxiv.org/abs/2401.17690v1","url_pdf":"https://arxiv.org/pdf/2401.17690v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"enclap-combining-neural-audio-codec-and-audio","repo_url":"https://github.com/jaeyeonkim99/enclap","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"audio-captioning","task_name":"Audio captioning"},{"task_slug":null,"task_name":"AudioCaps"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bart","method_name":"BART"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/audio-captioning-on-audiocaps","task":"Audio captioning","dataset":"AudioCaps","model":"EnCLAP-large","rank_in_archive_order":8,"of":18,"metrics":{"CIDEr":"0.8029","METEOR":"0.2554","SPICE":"0.1879","SPIDEr":"0.4954"},"uses_additional_data":false},{"leaderboard":"/sota/audio-captioning-on-audiocaps","task":"Audio captioning","dataset":"AudioCaps","model":"EnCLAP-base","rank_in_archive_order":10,"of":18,"metrics":{"CIDEr":"0.7795","METEOR":"0.2473","SPICE":"0.1863","SPIDEr":"0.4829"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2401.17690","atlas_url":"https://app.syntology.ai/?focus=2401.17690","mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}