{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/taming-visually-guided-sound-generation","title":"Taming Visually Guided Sound Generation","arxiv_id":"2110.08791","date":"2021-10-17","proceeding":null,"authors":["Vladimir Iashin","Esa Rahtu"],"abstract":"Recent advances in visually-induced audio generation are based on sampling short, low-fidelity, and one-class sounds. Moreover, sampling 1 second of audio from the state-of-the-art model takes minutes on a high-end GPU. In this work, we propose a single model capable of generating visually relevant, high-fidelity sounds prompted with a set of frames from open-domain videos in less time than it takes to play it on a single GPU. We train a transformer to sample a new spectrogram from the pre-trained spectrogram codebook given the set of video features. The codebook is obtained using a variant of VQGAN trained to produce a compact sampling space with a novel spectrogram-based perceptual loss. The generated spectrogram is transformed into a waveform using a window-based GAN that significantly speeds up generation. Considering the lack of metrics for automatic evaluation of generated spectrograms, we also build a family of metrics called FID and MKL. These metrics are based on a novel sound classifier, called Melception, and designed to evaluate the fidelity and relevance of open-domain samples. Both qualitative and quantitative studies are conducted on small- and large-scale datasets to evaluate the fidelity and relevance of generated samples. We also compare our model to the state-of-the-art and observe a substantial improvement in quality, size, and computation time. Code, demo, and samples: v-iashin.github.io/SpecVQGAN","url_abs":"https://arxiv.org/abs/2110.08791v1","url_pdf":"https://arxiv.org/pdf/2110.08791v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"taming-visually-guided-sound-generation","repo_url":"https://github.com/v-iashin/SpecVQGAN","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"taming-visually-guided-sound-generation","repo_url":"https://github.com/v-iashin/sparsesync","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"taming-visually-guided-sound-generation","repo_url":"https://github.com/v-iashin/video_features","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"audio-generation","task_name":"Audio Generation"},{"task_slug":null,"task_name":"GPU"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"cosine-annealing","method_name":"Cosine Annealing"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt-2","method_name":"GPT-2"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-cosine-annealing","method_name":"Linear Warmup With Cosine Annealing"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"vq-vae","method_name":"VQ-VAE"},{"method_slug":"weight-decay","method_name":"Weight Decay"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2110.08791","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.08791"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/v-iashin/sparsesync","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/v-iashin/video_features","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/v-iashin/SpecVQGAN","reach":null}],"summary":{"ran_draft_wrong":1,"unverified":6},"by_repo_kind":{"official":{"samples":1,"ran":1,"repositories":1},"listed":{"samples":6,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"4ff2d97e2d5610d9","entry":"patch_cfg_for_new_paths","repo":"v-iashin/SpecVQGAN","repo_kind":"official","path":"evaluate.py","file_url":"https://github.com/v-iashin/SpecVQGAN/blob/HEAD/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4ff2d97e2d5610d9"}},{"code_sha256_prefix":"d8286f360f701d2e","entry":"center_crop","repo":"v-iashin/video_features","repo_kind":"listed","path":"models/transforms.py","file_url":"https://github.com/v-iashin/video_features/blob/HEAD/models/transforms.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d8286f360f701d2e"}},{"code_sha256_prefix":"713a4fd814e4cfd5","entry":"crop","repo":"v-iashin/video_features","repo_kind":"listed","path":"models/transforms.py","file_url":"https://github.com/v-iashin/video_features/blob/HEAD/models/transforms.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"713a4fd814e4cfd5"}},{"code_sha256_prefix":"b65573125c5eb0f4","entry":"frame","repo":"v-iashin/video_features","repo_kind":"listed","path":"models/vggish/vggish_src/mel_features.py","file_url":"https://github.com/v-iashin/video_features/blob/HEAD/models/vggish/vggish_src/mel_features.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b65573125c5eb0f4"}},{"code_sha256_prefix":"0ffeb1da7ad6e62e","entry":"hflip","repo":"v-iashin/video_features","repo_kind":"listed","path":"models/transforms.py","file_url":"https://github.com/v-iashin/video_features/blob/HEAD/models/transforms.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0ffeb1da7ad6e62e"}},{"code_sha256_prefix":"29345538d55235c3","entry":"periodic_hann","repo":"v-iashin/video_features","repo_kind":"listed","path":"models/vggish/vggish_src/mel_features.py","file_url":"https://github.com/v-iashin/video_features/blob/HEAD/models/vggish/vggish_src/mel_features.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"29345538d55235c3"}},{"code_sha256_prefix":"bcec5b9bde143277","entry":"stft_magnitude","repo":"v-iashin/video_features","repo_kind":"listed","path":"models/vggish/vggish_src/mel_features.py","file_url":"https://github.com/v-iashin/video_features/blob/HEAD/models/vggish/vggish_src/mel_features.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bcec5b9bde143277"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}