{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/recap-retrieval-augmented-audio-captioning","title":"RECAP: Retrieval-Augmented Audio Captioning","arxiv_id":"2309.09836","date":"2023-09-18","proceeding":null,"authors":["Sreyan Ghosh","Sonal Kumar","Chandra Kiran Reddy Evuru","Ramani Duraiswami","Dinesh Manocha"],"abstract":"We present RECAP (REtrieval-Augmented Audio CAPtioning), a novel and effective audio captioning system that generates captions conditioned on an input audio and other captions similar to the audio retrieved from a datastore. Additionally, our proposed method can transfer to any domain without the need for any additional fine-tuning. To generate a caption for an audio sample, we leverage an audio-text model CLAP to retrieve captions similar to it from a replaceable datastore, which are then used to construct a prompt. Next, we feed this prompt to a GPT-2 decoder and introduce cross-attention layers between the CLAP encoder and GPT-2 to condition the audio for caption generation. Experiments on two benchmark datasets, Clotho and AudioCaps, show that RECAP achieves competitive performance in in-domain settings and significant improvements in out-of-domain settings. Additionally, due to its capability to exploit a large text-captions-only datastore in a training-free fashion, RECAP shows unique capabilities of captioning novel audio events never seen during training and compositional audios with multiple events. To promote research in this space, we also release 150,000+ new weakly labeled captions for AudioSet, AudioCaps, and Clotho.","url_abs":"https://arxiv.org/abs/2309.09836v2","url_pdf":"https://arxiv.org/pdf/2309.09836v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"recap-retrieval-augmented-audio-captioning","repo_url":"https://github.com/sreyan88/recap","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"audio-captioning","task_name":"Audio captioning"},{"task_slug":null,"task_name":"AudioCaps"},{"task_slug":"caption-generation","task_name":"Caption Generation"},{"task_slug":"decoder","task_name":"Decoder"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"retrieval-augmented-few-shot-in-context-audio","task_name":"Retrieval-augmented Few-shot In-context Audio Captioning"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"cosine-annealing","method_name":"Cosine Annealing"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"discriminative-fine-tuning","method_name":"Discriminative Fine-Tuning"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt-2","method_name":"GPT-2"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-cosine-annealing","method_name":"Linear Warmup With Cosine Annealing"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/retrieval-augmented-few-shot-in-context-audio","task":"Retrieval-augmented Few-shot In-context Audio Captioning","dataset":"AudioCaps","model":"RECAP (4-shot)","rank_in_archive_order":2,"of":5,"metrics":{"CIDEr":"0.359"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2309.09836","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2309.09836"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sreyan88/recap","reach":{"status":"ok"}}],"summary":{"ran":5,"unverified":3},"by_repo_kind":{"official":{"samples":8,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":8,"samples":[{"code_sha256_prefix":"829f97238622b090","entry":"load_clotho_data","repo":"sreyan88/recap","repo_kind":"official","path":"src/retrieve_caps_audio.py","file_url":"https://github.com/sreyan88/recap/blob/HEAD/src/retrieve_caps_audio.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"829f97238622b090"}},{"code_sha256_prefix":"1f5e94b6d75c2a8d","entry":"load_data_for_training","repo":"sreyan88/recap","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/sreyan88/recap/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1f5e94b6d75c2a8d"}},{"code_sha256_prefix":"b5b813d146850cb9","entry":"postprocess_preds","repo":"sreyan88/recap","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/sreyan88/recap/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b5b813d146850cb9"}},{"code_sha256_prefix":"f70cd7f51fd7d3e8","entry":"prep_strings","repo":"sreyan88/recap","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/sreyan88/recap/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f70cd7f51fd7d3e8"}},{"code_sha256_prefix":"e8d638fc7fcbd760","entry":"shift_tokens_right","repo":"sreyan88/recap","repo_kind":"official","path":"src/audio_encoder_decoder.py","file_url":"https://github.com/sreyan88/recap/blob/HEAD/src/audio_encoder_decoder.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e8d638fc7fcbd760"}},{"code_sha256_prefix":"769ba8b7eb1c2cc5","entry":"encode_captions","repo":"sreyan88/recap","repo_kind":"official","path":"src/retrieve_caps_audio.py","file_url":"https://github.com/sreyan88/recap/blob/HEAD/src/retrieve_caps_audio.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"769ba8b7eb1c2cc5"}},{"code_sha256_prefix":"3ff2fd0d68caa87b","entry":"filter_captions","repo":"sreyan88/recap","repo_kind":"official","path":"src/retrieve_caps_audio.py","file_url":"https://github.com/sreyan88/recap/blob/HEAD/src/retrieve_caps_audio.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3ff2fd0d68caa87b"}},{"code_sha256_prefix":"7f44f9a5502ba96a","entry":"load_model","repo":"sreyan88/recap","repo_kind":"official","path":"infer.py","file_url":"https://github.com/sreyan88/recap/blob/HEAD/infer.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7f44f9a5502ba96a"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}