{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/audiotoken-adaptation-of-text-conditioned-1","title":"AudioToken: Adaptation of Text-Conditioned Diffusion Models for Audio-to-Image Generation","arxiv_id":"2305.13050","date":"2023-05-22","proceeding":"Interspeech 2023 5","authors":["Guy Yariv","Itai Gat","Lior Wolf","Yossi Adi","Idan Schwartz"],"abstract":"In recent years, image generation has shown a great leap in performance, where diffusion models play a central role. Although generating high-quality images, such models are mainly conditioned on textual descriptions. This begs the question: \"how can we adopt such models to be conditioned on other modalities?\". In this paper, we propose a novel method utilizing latent diffusion models trained for text-to-image-generation to generate images conditioned on audio recordings. Using a pre-trained audio encoding model, the proposed method encodes audio into a new token, which can be considered as an adaptation layer between the audio and text representations. Such a modeling paradigm requires a small number of trainable parameters, making the proposed approach appealing for lightweight optimization. Results suggest the proposed method is superior to the evaluated baseline methods, considering objective and subjective metrics. Code and samples are available at: https://pages.cs.huji.ac.il/adiyoss-lab/AudioToken.","url_abs":"https://arxiv.org/abs/2305.13050v1","url_pdf":"https://arxiv.org/pdf/2305.13050v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"audiotoken-adaptation-of-text-conditioned-1","repo_url":"https://github.com/guyyariv/AudioToken","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"audiotoken-adaptation-of-text-conditioned-1","repo_url":"https://github.com/guyyariv/vlmig","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"image-generation","task_name":"Image Generation"},{"task_slug":"text-to-image-generation-1","task_name":"Text to Image Generation"},{"task_slug":"text-to-image-generation","task_name":"Text-to-Image Generation"},{"task_slug":"audio-visual-learning","task_name":"audio-visual learning"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2305.13050","atlas_url":"https://app.syntology.ai/?focus=2305.13050","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.13050"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/guyyariv/AudioToken","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/guyyariv/vlmig","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":3,"ran_fixture":1,"unverified":7},"by_repo_kind":{"official":{"samples":8,"ran":4,"repositories":1},"listed":{"samples":3,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"4675779aec92d6c0","entry":"contrastive_loss","repo":"guyyariv/AudioToken","repo_kind":"official","path":"modules/clip_text_model/modeling_clip.py","file_url":"https://github.com/guyyariv/AudioToken/blob/HEAD/modules/clip_text_model/modeling_clip.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4675779aec92d6c0"}},{"code_sha256_prefix":"f30ebf4e48b7e2d7","entry":"gelu","repo":"guyyariv/AudioToken","repo_kind":"official","path":"modules/BEATs/modules.py","file_url":"https://github.com/guyyariv/AudioToken/blob/HEAD/modules/BEATs/modules.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f30ebf4e48b7e2d7"}},{"code_sha256_prefix":"eca6cdf05972a95e","entry":"l2norm","repo":"guyyariv/AudioToken","repo_kind":"official","path":"modules/BEATs/quantizer.py","file_url":"https://github.com/guyyariv/AudioToken/blob/HEAD/modules/BEATs/quantizer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"eca6cdf05972a95e"}},{"code_sha256_prefix":"ada1d74afbd92171","entry":"sample_vectors","repo":"guyyariv/AudioToken","repo_kind":"official","path":"modules/BEATs/quantizer.py","file_url":"https://github.com/guyyariv/AudioToken/blob/HEAD/modules/BEATs/quantizer.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ada1d74afbd92171"}},{"code_sha256_prefix":"f2ca34eca05d40cd","entry":"clip_loss","repo":"guyyariv/AudioToken","repo_kind":"official","path":"modules/clip_text_model/modeling_clip.py","file_url":"https://github.com/guyyariv/AudioToken/blob/HEAD/modules/clip_text_model/modeling_clip.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f2ca34eca05d40cd"}},{"code_sha256_prefix":"4aa7333ffd7e18c6","entry":"gelu_accurate","repo":"guyyariv/AudioToken","repo_kind":"official","path":"modules/BEATs/modules.py","file_url":"https://github.com/guyyariv/AudioToken/blob/HEAD/modules/BEATs/modules.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4aa7333ffd7e18c6"}},{"code_sha256_prefix":"01233f4856456c5f","entry":"get_activation_fn","repo":"guyyariv/AudioToken","repo_kind":"official","path":"modules/BEATs/modules.py","file_url":"https://github.com/guyyariv/AudioToken/blob/HEAD/modules/BEATs/modules.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"01233f4856456c5f"}},{"code_sha256_prefix":"b865ae3ee66b52d3","entry":"kmeans","repo":"guyyariv/AudioToken","repo_kind":"official","path":"modules/BEATs/quantizer.py","file_url":"https://github.com/guyyariv/AudioToken/blob/HEAD/modules/BEATs/quantizer.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b865ae3ee66b52d3"}},{"code_sha256_prefix":"5779075b2c0609d6","entry":"load_cropped_vg_regions","repo":"guyyariv/vlmig","repo_kind":"listed","path":"load_datasets.py","file_url":"https://github.com/guyyariv/vlmig/blob/HEAD/load_datasets.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5779075b2c0609d6"}},{"code_sha256_prefix":"09a58b20cdc7cc72","entry":"load_laion_220","repo":"guyyariv/vlmig","repo_kind":"listed","path":"load_datasets.py","file_url":"https://github.com/guyyariv/vlmig/blob/HEAD/load_datasets.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"09a58b20cdc7cc72"}},{"code_sha256_prefix":"068213d67698b514","entry":"load_wiki","repo":"guyyariv/vlmig","repo_kind":"listed","path":"load_datasets.py","file_url":"https://github.com/guyyariv/vlmig/blob/HEAD/load_datasets.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"068213d67698b514"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}