{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/uavm-a-unified-model-for-audio-visual","title":"UAVM: Towards Unifying Audio and Visual Models","arxiv_id":"2208.00061","date":"2022-07-29","proceeding":null,"authors":["Yuan Gong","Alexander H. Liu","Andrew Rouditchenko","James Glass"],"abstract":"Conventional audio-visual models have independent audio and video branches. In this work, we unify the audio and visual branches by designing a Unified Audio-Visual Model (UAVM). The UAVM achieves a new state-of-the-art audio-visual event classification accuracy of 65.8% on VGGSound. More interestingly, we also find a few intriguing properties of UAVM that the modality-independent counterparts do not have.","url_abs":"https://arxiv.org/abs/2208.00061v2","url_pdf":"https://arxiv.org/pdf/2208.00061v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"uavm-a-unified-model-for-audio-visual","repo_url":"https://github.com/YuanGongND/uavm","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"BSD-2-Clause"}}],"tasks":[{"task_slug":"audio-classification","task_name":"Audio Classification"},{"task_slug":"multi-modal-classification","task_name":"Multi-modal Classification"},{"task_slug":"audio-visual-learning","task_name":"audio-visual learning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/audio-classification-on-audioset","task":"Audio Classification","dataset":"AudioSet","model":"UAVM (Audio + Video)","rank_in_archive_order":8,"of":51,"metrics":{"Test mAP":"0.504"},"uses_additional_data":true},{"leaderboard":"/sota/audio-classification-on-vggsound","task":"Audio Classification","dataset":"VGGSound","model":"UAVM (Audio + Video)","rank_in_archive_order":9,"of":23,"metrics":{"Top 1 Accuracy":"65.8"},"uses_additional_data":true},{"leaderboard":"/sota/audio-classification-on-vggsound","task":"Audio Classification","dataset":"VGGSound","model":"UAVM (Audio Only)","rank_in_archive_order":16,"of":23,"metrics":{"Top 1 Accuracy":"56.5"},"uses_additional_data":true},{"leaderboard":"/sota/audio-classification-on-vggsound","task":"Audio Classification","dataset":"VGGSound","model":"UAVM (Video Only)","rank_in_archive_order":22,"of":23,"metrics":{"Top 1 Accuracy":"49.9"},"uses_additional_data":true},{"leaderboard":"/sota/multi-modal-classification-on-audioset","task":"Multi-modal Classification","dataset":"AudioSet","model":"UAVM","rank_in_archive_order":2,"of":2,"metrics":{"Average mAP":"0.504"},"uses_additional_data":true},{"leaderboard":"/sota/multi-modal-classification-on-vgg-sound","task":"Multi-modal Classification","dataset":"VGG-Sound","model":"UAVM","rank_in_archive_order":3,"of":4,"metrics":{"Top-1 Accuracy":"65.8"},"uses_additional_data":true}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2208.00061","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2208.00061"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/YuanGongND/uavm","reach":{"status":"ok","spdx":"BSD-2-Clause"}}],"summary":{"ran":6,"unverified":8},"by_repo_kind":{"official":{"samples":14,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"e217fb47bef3abe4","entry":"get_immediate_subdirectories","repo":"YuanGongND/uavm","repo_kind":"official","path":"src/attention_diff/gen_att_map_vggsound_full_att_summary.py","file_url":"https://github.com/YuanGongND/uavm/blob/HEAD/src/attention_diff/gen_att_map_vggsound_full_att_summary.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"e217fb47bef3abe4"}},{"code_sha256_prefix":"72584682b61aedb0","entry":"get_sim_mat","repo":"YuanGongND/uavm","repo_kind":"official","path":"src/attention_diff/gen_att_map_vggsound_full_att_summary.py","file_url":"https://github.com/YuanGongND/uavm/blob/HEAD/src/attention_diff/gen_att_map_vggsound_full_att_summary.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"72584682b61aedb0"}},{"code_sha256_prefix":"04fa3bcbfec2c0c2","entry":"get_similarity","repo":"YuanGongND/uavm","repo_kind":"official","path":"src/attention_diff/gen_att_map_vggsound_full_att_summary.py","file_url":"https://github.com/YuanGongND/uavm/blob/HEAD/src/attention_diff/gen_att_map_vggsound_full_att_summary.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"04fa3bcbfec2c0c2"}},{"code_sha256_prefix":"c15efdd12442aeb4","entry":"lookup_list","repo":"YuanGongND/uavm","repo_kind":"official","path":"src/dataloader.py","file_url":"https://github.com/YuanGongND/uavm/blob/HEAD/src/dataloader.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"c15efdd12442aeb4"}},{"code_sha256_prefix":"a8fb3f62e2a901d7","entry":"make_index_dict","repo":"YuanGongND/uavm","repo_kind":"official","path":"src/dataloader.py","file_url":"https://github.com/YuanGongND/uavm/blob/HEAD/src/dataloader.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"a8fb3f62e2a901d7"}},{"code_sha256_prefix":"cbf115a314bcd23e","entry":"make_name_dict","repo":"YuanGongND/uavm","repo_kind":"official","path":"src/dataloader.py","file_url":"https://github.com/YuanGongND/uavm/blob/HEAD/src/dataloader.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"cbf115a314bcd23e"}},{"code_sha256_prefix":"43c80235310e3600","entry":"cal_modality_cla","repo":"YuanGongND/uavm","repo_kind":"official","path":"src/embedding/check_modality_classification_summary.py","file_url":"https://github.com/YuanGongND/uavm/blob/HEAD/src/embedding/check_modality_classification_summary.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"43c80235310e3600"}},{"code_sha256_prefix":"9ff67c442329e8a7","entry":"get_result","repo":"YuanGongND/uavm","repo_kind":"official","path":"src/attention_diff/plt_att_diff.py","file_url":"https://github.com/YuanGongND/uavm/blob/HEAD/src/attention_diff/plt_att_diff.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"9ff67c442329e8a7"}},{"code_sha256_prefix":"61299eb1ed3c706c","entry":"load_files","repo":"YuanGongND/uavm","repo_kind":"official","path":"src/attention_diff/plt_att_diff.py","file_url":"https://github.com/YuanGongND/uavm/blob/HEAD/src/attention_diff/plt_att_diff.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"61299eb1ed3c706c"}},{"code_sha256_prefix":"331bb441f92a339c","entry":"modality_cla","repo":"YuanGongND/uavm","repo_kind":"official","path":"src/embedding/check_modality_classification_summary.py","file_url":"https://github.com/YuanGongND/uavm/blob/HEAD/src/embedding/check_modality_classification_summary.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"331bb441f92a339c"}},{"code_sha256_prefix":"8c4b5afbcd37a80f","entry":"validate","repo":"YuanGongND/uavm","repo_kind":"official","path":"src/traintest.py","file_url":"https://github.com/YuanGongND/uavm/blob/HEAD/src/traintest.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"8c4b5afbcd37a80f"}},{"code_sha256_prefix":"cc7e8ccff12882a7","entry":"validate","repo":"YuanGongND/uavm","repo_kind":"official","path":"src/traintest_feat.py","file_url":"https://github.com/YuanGongND/uavm/blob/HEAD/src/traintest_feat.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"cc7e8ccff12882a7"}},{"code_sha256_prefix":"29f6d6e61e31cfd1","entry":"validate_ensemble","repo":"YuanGongND/uavm","repo_kind":"official","path":"src/traintest_feat.py","file_url":"https://github.com/YuanGongND/uavm/blob/HEAD/src/traintest_feat.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"29f6d6e61e31cfd1"}},{"code_sha256_prefix":"e2fc9876fd34a120","entry":"validate_wa","repo":"YuanGongND/uavm","repo_kind":"official","path":"src/traintest_feat.py","file_url":"https://github.com/YuanGongND/uavm/blob/HEAD/src/traintest_feat.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"e2fc9876fd34a120"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}