{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/conditional-generation-of-audio-from-video","title":"Conditional Generation of Audio from Video via Foley Analogies","arxiv_id":"2304.08490","date":"2023-04-17","proceeding":"CVPR 2023 1","authors":["Yuexi Du","Ziyang Chen","Justin Salamon","Bryan Russell","Andrew Owens"],"abstract":"The sound effects that designers add to videos are designed to convey a particular artistic effect and, thus, may be quite different from a scene's true sound. Inspired by the challenges of creating a soundtrack for a video that differs from its true sound, but that nonetheless matches the actions occurring on screen, we propose the problem of conditional Foley. We present the following contributions to address this problem. First, we propose a pretext task for training our model to predict sound for an input video clip using a conditional audio-visual clip sampled from another time within the same source video. Second, we propose a model for generating a soundtrack for a silent input video, given a user-supplied example that specifies what the video should \"sound like\". We show through human studies and automated evaluation metrics that our model successfully generates sound from video, while varying its output according to the content of a supplied example. Project site: https://xypb.github.io/CondFoleyGen/","url_abs":"https://arxiv.org/abs/2304.08490v1","url_pdf":"https://arxiv.org/pdf/2304.08490v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"conditional-generation-of-audio-from-video","repo_url":"https://github.com/XYPB/CondFoleyGen","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[],"methods":[{"method_slug":"clip","method_name":"CLIP"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2304.08490","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2304.08490"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/XYPB/CondFoleyGen","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":1,"unverified":12},"by_repo_kind":{"official":{"samples":13,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"1090950b58e9358b","entry":"gram_matrix","repo":"XYPB/CondFoleyGen","repo_kind":"official","path":"audio_generation.py","file_url":"https://github.com/XYPB/CondFoleyGen/blob/HEAD/audio_generation.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1090950b58e9358b"}},{"code_sha256_prefix":"e2037f0f6ae0a1bc","entry":"check_match","repo":"XYPB/CondFoleyGen","repo_kind":"official","path":"audio_type_analysis.py","file_url":"https://github.com/XYPB/CondFoleyGen/blob/HEAD/audio_type_analysis.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e2037f0f6ae0a1bc"}},{"code_sha256_prefix":"707cb66d1f960dda","entry":"create_gen_key2orig_key","repo":"XYPB/CondFoleyGen","repo_kind":"official","path":"create_test_folder.py","file_url":"https://github.com/XYPB/CondFoleyGen/blob/HEAD/create_test_folder.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"707cb66d1f960dda"}},{"code_sha256_prefix":"8c86f518bf3ee01a","entry":"get_style_model_and_losses","repo":"XYPB/CondFoleyGen","repo_kind":"official","path":"audio_generation.py","file_url":"https://github.com/XYPB/CondFoleyGen/blob/HEAD/audio_generation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8c86f518bf3ee01a"}},{"code_sha256_prefix":"295b08649ff00eba","entry":"load_specs_as_img","repo":"XYPB/CondFoleyGen","repo_kind":"official","path":"audio_generation.py","file_url":"https://github.com/XYPB/CondFoleyGen/blob/HEAD/audio_generation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"295b08649ff00eba"}},{"code_sha256_prefix":"3f45383acb251f26","entry":"make_folder_for_samples","repo":"XYPB/CondFoleyGen","repo_kind":"official","path":"evaluation/generate_samples.py","file_url":"https://github.com/XYPB/CondFoleyGen/blob/HEAD/evaluation/generate_samples.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3f45383acb251f26"}},{"code_sha256_prefix":"6609e7e473de823b","entry":"onset_nms","repo":"XYPB/CondFoleyGen","repo_kind":"official","path":"predict_onset.py","file_url":"https://github.com/XYPB/CondFoleyGen/blob/HEAD/predict_onset.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6609e7e473de823b"}},{"code_sha256_prefix":"14890f5890893ac8","entry":"pipeline_extract_audio","repo":"XYPB/CondFoleyGen","repo_kind":"official","path":"video_preprocess.py","file_url":"https://github.com/XYPB/CondFoleyGen/blob/HEAD/video_preprocess.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"14890f5890893ac8"}},{"code_sha256_prefix":"4473929782c8d1c1","entry":"pipeline_reencode_mov","repo":"XYPB/CondFoleyGen","repo_kind":"official","path":"video_preprocess.py","file_url":"https://github.com/XYPB/CondFoleyGen/blob/HEAD/video_preprocess.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4473929782c8d1c1"}},{"code_sha256_prefix":"ca2c953686f5cb93","entry":"pipeline_resize","repo":"XYPB/CondFoleyGen","repo_kind":"official","path":"video_preprocess.py","file_url":"https://github.com/XYPB/CondFoleyGen/blob/HEAD/video_preprocess.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ca2c953686f5cb93"}},{"code_sha256_prefix":"3f48d259e25d02bc","entry":"process_action","repo":"XYPB/CondFoleyGen","repo_kind":"official","path":"audio_type_analysis.py","file_url":"https://github.com/XYPB/CondFoleyGen/blob/HEAD/audio_type_analysis.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3f48d259e25d02bc"}},{"code_sha256_prefix":"cb8eaae03e9b85ac","entry":"process_type","repo":"XYPB/CondFoleyGen","repo_kind":"official","path":"audio_type_analysis.py","file_url":"https://github.com/XYPB/CondFoleyGen/blob/HEAD/audio_type_analysis.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cb8eaae03e9b85ac"}},{"code_sha256_prefix":"48ef389d993cb3c8","entry":"sample_spectrogram","repo":"XYPB/CondFoleyGen","repo_kind":"official","path":"evaluation/generate_samples.py","file_url":"https://github.com/XYPB/CondFoleyGen/blob/HEAD/evaluation/generate_samples.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"48ef389d993cb3c8"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}