{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/clotho-an-audio-captioning-dataset","title":"Clotho: An Audio Captioning Dataset","arxiv_id":"1910.09387","date":"2019-10-21","proceeding":null,"authors":["Konstantinos Drossos","Samuel Lipping","Tuomas Virtanen"],"abstract":"Audio captioning is the novel task of general audio content description using free text. It is an intermodal translation task (not speech-to-text), where a system accepts as an input an audio signal and outputs the textual description (i.e. the caption) of that signal. In this paper we present Clotho, a dataset for audio captioning consisting of 4981 audio samples of 15 to 30 seconds duration and 24 905 captions of eight to 20 words length, and a baseline method to provide initial results. Clotho is built with focus on audio content and caption diversity, and the splits of the data are not hampering the training or evaluation of methods. All sounds are from the Freesound platform, and captions are crowdsourced using Amazon Mechanical Turk and annotators from English speaking countries. Unique words, named entities, and speech transcription are removed with post-processing. Clotho is freely available online (https://zenodo.org/record/3490684).","url_abs":"https://arxiv.org/abs/1910.09387v1","url_pdf":"https://arxiv.org/pdf/1910.09387v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"clotho-an-audio-captioning-dataset","repo_url":"https://github.com/akoepke/audio-retrieval-benchmark","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"clotho-an-audio-captioning-dataset","repo_url":"https://github.com/audio-captioning/clotho-dataset","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"clotho-an-audio-captioning-dataset","repo_url":"https://github.com/dr-costas/clotho-baseline-dataset","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"clotho-an-audio-captioning-dataset","repo_url":"https://github.com/labbeti/aac-datasets","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"clotho-an-audio-captioning-dataset","repo_url":"https://github.com/oncescuandreea/audio-retrieval","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"clotho-an-audio-captioning-dataset","repo_url":"https://github.com/richermans/AudioCaption","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"clotho-an-audio-captioning-dataset","repo_url":"https://github.com/wsntxxn/AudioCaption","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"audio-captioning","task_name":"Audio captioning"},{"task_slug":"diversity","task_name":"Diversity"},{"task_slug":"speech-to-text","task_name":"Speech-to-Text"},{"task_slug":"translation","task_name":"Translation"}],"methods":[],"datasets_introduced":[{"slug":"clotho","name":"Clotho","full_name":"Clotho"}],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/1910.09387","atlas_url":"https://app.syntology.ai/?focus=1910.09387","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1910.09387"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/labbeti/aac-datasets","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/audio-captioning/clotho-dataset","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/wsntxxn/AudioCaption","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/richermans/AudioCaption","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/akoepke/audio-retrieval-benchmark","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/oncescuandreea/audio-retrieval","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/dr-costas/clotho-baseline-dataset","reach":{"status":"ok","spdx":"NOASSERTION"}}],"summary":{"ran":14,"ran_draft_wrong":2,"unverified":5},"by_repo_kind":{"listed":{"samples":21,"ran":16,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"36732bf2fa117007","entry":"NAME_TO_WIDTH","repo":"wsntxxn/AudioCaption","repo_kind":"listed","path":"captioning/models/eff_at_encoder.py","file_url":"https://github.com/wsntxxn/AudioCaption/blob/HEAD/captioning/models/eff_at_encoder.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"36732bf2fa117007"}},{"code_sha256_prefix":"96fd7d5df7f15091","entry":"cnn_out_size","repo":"wsntxxn/AudioCaption","repo_kind":"listed","path":"captioning/models/eff_at_encoder.py","file_url":"https://github.com/wsntxxn/AudioCaption/blob/HEAD/captioning/models/eff_at_encoder.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"96fd7d5df7f15091"}},{"code_sha256_prefix":"7c55804fff5d4cc0","entry":"conv_conv_block","repo":"wsntxxn/AudioCaption","repo_kind":"listed","path":"captioning/models/crnn_trm_encoder.py","file_url":"https://github.com/wsntxxn/AudioCaption/blob/HEAD/captioning/models/crnn_trm_encoder.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7c55804fff5d4cc0"}},{"code_sha256_prefix":"8ebc0d04df3640f5","entry":"cs_interspeech","repo":"wsntxxn/AudioCaption","repo_kind":"listed","path":"captioning/models/conv_filter_pruning.py","file_url":"https://github.com/wsntxxn/AudioCaption/blob/HEAD/captioning/models/conv_filter_pruning.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8ebc0d04df3640f5"}},{"code_sha256_prefix":"99b3ce923e424db3","entry":"generate_length_mask","repo":"wsntxxn/AudioCaption","repo_kind":"listed","path":"captioning/models/flops_counting_model.py","file_url":"https://github.com/wsntxxn/AudioCaption/blob/HEAD/captioning/models/flops_counting_model.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"99b3ce923e424db3"}},{"code_sha256_prefix":"e698f855f3588740","entry":"gumbel_softmax_sample","repo":"wsntxxn/AudioCaption","repo_kind":"listed","path":"captioning/losses/loss.py","file_url":"https://github.com/wsntxxn/AudioCaption/blob/HEAD/captioning/losses/loss.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e698f855f3588740"}},{"code_sha256_prefix":"9e10f323aeefe252","entry":"make_divisible","repo":"wsntxxn/AudioCaption","repo_kind":"listed","path":"captioning/models/eff_at_encoder.py","file_url":"https://github.com/wsntxxn/AudioCaption/blob/HEAD/captioning/models/eff_at_encoder.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9e10f323aeefe252"}},{"code_sha256_prefix":"df07b4faa7609d90","entry":"open_or_fd","repo":"richermans/AudioCaption","repo_kind":"listed","path":"utils/kaldi_io.py","file_url":"https://github.com/richermans/AudioCaption/blob/HEAD/utils/kaldi_io.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"df07b4faa7609d90"}},{"code_sha256_prefix":"18d5889c7ea06109","entry":"operator_norm_pruning","repo":"wsntxxn/AudioCaption","repo_kind":"listed","path":"captioning/models/conv_filter_pruning.py","file_url":"https://github.com/wsntxxn/AudioCaption/blob/HEAD/captioning/models/conv_filter_pruning.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"18d5889c7ea06109"}},{"code_sha256_prefix":"bfac58a04b6835f5","entry":"pad_unsort_packed_sequence","repo":"wsntxxn/AudioCaption","repo_kind":"listed","path":"captioning/models/hf_wrapper.py","file_url":"https://github.com/wsntxxn/AudioCaption/blob/HEAD/captioning/models/hf_wrapper.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bfac58a04b6835f5"}},{"code_sha256_prefix":"d9a4862872e34aad","entry":"popen","repo":"richermans/AudioCaption","repo_kind":"listed","path":"utils/kaldi_io.py","file_url":"https://github.com/richermans/AudioCaption/blob/HEAD/utils/kaldi_io.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d9a4862872e34aad"}},{"code_sha256_prefix":"0fac7426325be231","entry":"rank1_apporx","repo":"wsntxxn/AudioCaption","repo_kind":"listed","path":"captioning/models/conv_filter_pruning.py","file_url":"https://github.com/wsntxxn/AudioCaption/blob/HEAD/captioning/models/conv_filter_pruning.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0fac7426325be231"}},{"code_sha256_prefix":"aa10e48fc468d9e0","entry":"read_key","repo":"richermans/AudioCaption","repo_kind":"listed","path":"utils/kaldi_io.py","file_url":"https://github.com/richermans/AudioCaption/blob/HEAD/utils/kaldi_io.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"aa10e48fc468d9e0"}},{"code_sha256_prefix":"5ead97ccfc2dee7b","entry":"reparameterize_argmax","repo":"wsntxxn/AudioCaption","repo_kind":"listed","path":"captioning/losses/loss.py","file_url":"https://github.com/wsntxxn/AudioCaption/blob/HEAD/captioning/losses/loss.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5ead97ccfc2dee7b"}},{"code_sha256_prefix":"2908a8bdfec77e29","entry":"sample_gumbel","repo":"wsntxxn/AudioCaption","repo_kind":"listed","path":"captioning/losses/loss.py","file_url":"https://github.com/wsntxxn/AudioCaption/blob/HEAD/captioning/losses/loss.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2908a8bdfec77e29"}},{"code_sha256_prefix":"86839e01585b85d8","entry":"sort_pack_padded_sequence","repo":"wsntxxn/AudioCaption","repo_kind":"listed","path":"captioning/models/hf_wrapper.py","file_url":"https://github.com/wsntxxn/AudioCaption/blob/HEAD/captioning/models/hf_wrapper.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"86839e01585b85d8"}},{"code_sha256_prefix":"ef7046e8db3256f6","entry":"build_vocab","repo":"richermans/AudioCaption","repo_kind":"listed","path":"utils/build_vocab.py","file_url":"https://github.com/richermans/AudioCaption/blob/HEAD/utils/build_vocab.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ef7046e8db3256f6"}},{"code_sha256_prefix":"42b965c67572da92","entry":"coco_score","repo":"richermans/AudioCaption","repo_kind":"listed","path":"utils/human_eval.py","file_url":"https://github.com/richermans/AudioCaption/blob/HEAD/utils/human_eval.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"42b965c67572da92"}},{"code_sha256_prefix":"47fe80f824147b4d","entry":"collate_fn","repo":"richermans/AudioCaption","repo_kind":"listed","path":"datasets/SJTUDataSet.py","file_url":"https://github.com/richermans/AudioCaption/blob/HEAD/datasets/SJTUDataSet.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"47fe80f824147b4d"}},{"code_sha256_prefix":"471b42e75608e3d8","entry":"pack_wrapper","repo":"wsntxxn/AudioCaption","repo_kind":"listed","path":"captioning/models/hf_wrapper.py","file_url":"https://github.com/wsntxxn/AudioCaption/blob/HEAD/captioning/models/hf_wrapper.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"471b42e75608e3d8"}},{"code_sha256_prefix":"c26450c99bacd017","entry":"parse_poolingfunction","repo":"richermans/AudioCaption","repo_kind":"listed","path":"models/encoder.py","file_url":"https://github.com/richermans/AudioCaption/blob/HEAD/models/encoder.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c26450c99bacd017"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}