{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/equiav-leveraging-equivariance-for-audio","title":"EquiAV: Leveraging Equivariance for Audio-Visual Contrastive Learning","arxiv_id":"2403.09502","date":"2024-03-14","proceeding":null,"authors":["Jongsuk Kim","Hyeongkeun Lee","Kyeongha Rho","Junmo Kim","Joon Son Chung"],"abstract":"Recent advancements in self-supervised audio-visual representation learning have demonstrated its potential to capture rich and comprehensive representations. However, despite the advantages of data augmentation verified in many learning methods, audio-visual learning has struggled to fully harness these benefits, as augmentations can easily disrupt the correspondence between input pairs. To address this limitation, we introduce EquiAV, a novel framework that leverages equivariance for audio-visual contrastive learning. Our approach begins with extending equivariance to audio-visual learning, facilitated by a shared attention-based transformation predictor. It enables the aggregation of features from diverse augmentations into a representative embedding, providing robust supervision. Notably, this is achieved with minimal computational overhead. Extensive ablation studies and qualitative results verify the effectiveness of our method. EquiAV outperforms previous works across various audio-visual benchmarks. The code is available on https://github.com/JongSuk1/EquiAV.","url_abs":"https://arxiv.org/abs/2403.09502v2","url_pdf":"https://arxiv.org/pdf/2403.09502v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"equiav-leveraging-equivariance-for-audio","repo_url":"https://github.com/jongsuk1/equiav","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"audio-classification","task_name":"Audio Classification"},{"task_slug":"contrastive-learning","task_name":"Contrastive Learning"},{"task_slug":"data-augmentation","task_name":"Data Augmentation"},{"task_slug":"representation-learning","task_name":"Representation Learning"},{"task_slug":"audio-visual-learning","task_name":"audio-visual learning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/audio-classification-on-audioset","task":"Audio Classification","dataset":"AudioSet","model":"EquiAV","rank_in_archive_order":3,"of":51,"metrics":{"Test mAP":"0.546"},"uses_additional_data":false},{"leaderboard":"/sota/audio-classification-on-balanced-audio-set","task":"Audio Classification","dataset":"Balanced Audio Set","model":"EquiAV","rank_in_archive_order":1,"of":8,"metrics":{"Mean AP":"42.4"},"uses_additional_data":false},{"leaderboard":"/sota/audio-classification-on-vggsound","task":"Audio Classification","dataset":"VGGSound","model":"EquiAV","rank_in_archive_order":6,"of":23,"metrics":{"Top 1 Accuracy":"67.1"},"uses_additional_data":true}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2403.09502","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.09502"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/JongSuk1/EquiAV","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jongsuk1/equiav","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":6,"unverified":7},"by_repo_kind":{"official":{"samples":13,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"d4d9aba462915ffd","entry":"CrossAttention","repo":"jongsuk1/equiav","repo_kind":"official","path":"models/pt_EquiAV.py","file_url":"https://github.com/jongsuk1/equiav/blob/HEAD/models/pt_EquiAV.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d4d9aba462915ffd"}},{"code_sha256_prefix":"894dad2556348ddd","entry":"CrossAttentionBlock","repo":"jongsuk1/equiav","repo_kind":"official","path":"models/pt_EquiAV.py","file_url":"https://github.com/jongsuk1/equiav/blob/HEAD/models/pt_EquiAV.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"894dad2556348ddd"}},{"code_sha256_prefix":"f5a101a28eb19371","entry":"MLPProjectionHead","repo":"jongsuk1/equiav","repo_kind":"official","path":"models/pt_EquiAV.py","file_url":"https://github.com/jongsuk1/equiav/blob/HEAD/models/pt_EquiAV.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f5a101a28eb19371"}},{"code_sha256_prefix":"fe14abb5583a3609","entry":"PatchEmbed","repo":"jongsuk1/equiav","repo_kind":"official","path":"models/pt_EquiAV.py","file_url":"https://github.com/jongsuk1/equiav/blob/HEAD/models/pt_EquiAV.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fe14abb5583a3609"}},{"code_sha256_prefix":"d59b4b3c004c31ff","entry":"compute_metrics","repo":"JongSuk1/EquiAV","repo_kind":"official","path":"retrieval.py","file_url":"https://github.com/JongSuk1/EquiAV/blob/HEAD/retrieval.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d59b4b3c004c31ff"}},{"code_sha256_prefix":"a1ababfdce80565d","entry":"get_sim_mat","repo":"JongSuk1/EquiAV","repo_kind":"official","path":"retrieval.py","file_url":"https://github.com/JongSuk1/EquiAV/blob/HEAD/retrieval.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a1ababfdce80565d"}},{"code_sha256_prefix":"76fafb76150402b2","entry":"Block","repo":"jongsuk1/equiav","repo_kind":"official","path":"models/pt_EquiAV.py","file_url":"https://github.com/jongsuk1/equiav/blob/HEAD/models/pt_EquiAV.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"76fafb76150402b2"}},{"code_sha256_prefix":"291f78cc0fbb175c","entry":"MainModel","repo":"jongsuk1/equiav","repo_kind":"official","path":"models/pt_EquiAV.py","file_url":"https://github.com/jongsuk1/equiav/blob/HEAD/models/pt_EquiAV.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"291f78cc0fbb175c"}},{"code_sha256_prefix":"12035a2f77d8016c","entry":"get_1d_sincos_pos_embed_from_grid","repo":"JongSuk1/EquiAV","repo_kind":"official","path":"models/pos_embed.py","file_url":"https://github.com/JongSuk1/EquiAV/blob/HEAD/models/pos_embed.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"12035a2f77d8016c"}},{"code_sha256_prefix":"660403f722993455","entry":"get_2d_sincos_pos_embed","repo":"jongsuk1/equiav","repo_kind":"official","path":"models/pt_EquiAV.py","file_url":"https://github.com/jongsuk1/equiav/blob/HEAD/models/pt_EquiAV.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"660403f722993455"}},{"code_sha256_prefix":"57abad7bed6d36ff","entry":"get_2d_sincos_pos_embed","repo":"JongSuk1/EquiAV","repo_kind":"official","path":"models/pos_embed.py","file_url":"https://github.com/JongSuk1/EquiAV/blob/HEAD/models/pos_embed.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"57abad7bed6d36ff"}},{"code_sha256_prefix":"f10004e059714d42","entry":"get_2d_sincos_pos_embed_from_grid","repo":"JongSuk1/EquiAV","repo_kind":"official","path":"models/pos_embed.py","file_url":"https://github.com/JongSuk1/EquiAV/blob/HEAD/models/pos_embed.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f10004e059714d42"}},{"code_sha256_prefix":"cf8971124ae68c20","entry":"get_retrieval_result","repo":"JongSuk1/EquiAV","repo_kind":"official","path":"retrieval.py","file_url":"https://github.com/JongSuk1/EquiAV/blob/HEAD/retrieval.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cf8971124ae68c20"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}