{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/audio-visual-instance-discrimination-with","title":"Audio-Visual Instance Discrimination with Cross-Modal Agreement","arxiv_id":"2004.12943","date":"2020-04-27","proceeding":"CVPR 2021 1","authors":["Pedro Morgado","Nuno Vasconcelos","Ishan Misra"],"abstract":"We present a self-supervised learning approach to learn audio-visual representations from video and audio. Our method uses contrastive learning for cross-modal discrimination of video from audio and vice-versa. We show that optimizing for cross-modal discrimination, rather than within-modal discrimination, is important to learn good representations from video and audio. With this simple but powerful insight, our method achieves highly competitive performance when finetuned on action recognition tasks. Furthermore, while recent work in contrastive learning defines positive and negative samples as individual instances, we generalize this definition by exploring cross-modal agreement. We group together multiple instances as positives by measuring their similarity in both the video and audio feature spaces. Cross-modal agreement creates better positive and negative sets, which allows us to calibrate visual similarities by seeking within-modal discrimination of positive instances, and achieve significant gains on downstream tasks.","url_abs":"https://arxiv.org/abs/2004.12943v3","url_pdf":"https://arxiv.org/pdf/2004.12943v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"audio-visual-instance-discrimination-with","repo_url":"https://github.com/facebookresearch/AVID-CMA","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}}],"tasks":[{"task_slug":"action-recognition-in-videos","task_name":"Action Recognition"},{"task_slug":"audio-classification","task_name":"Audio Classification"},{"task_slug":"contrastive-learning","task_name":"Contrastive Learning"},{"task_slug":"self-supervised-action-recognition","task_name":"Self-Supervised Action Recognition"},{"task_slug":"self-supervised-audio-classification","task_name":"Self-Supervised Audio Classification"},{"task_slug":"self-supervised-learning","task_name":"Self-Supervised Learning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/audio-classification-on-esc-50","task":"Audio Classification","dataset":"ESC-50","model":"AVID","rank_in_archive_order":24,"of":29,"metrics":{"Top-1 Accuracy":"89.2"},"uses_additional_data":false},{"leaderboard":"/sota/self-supervised-action-recognition-on-hmdb51","task":"Self-Supervised Action Recognition","dataset":"HMDB51","model":"AVID+CMA (Modified R2+1D-18 on Audioset)","rank_in_archive_order":16,"of":48,"metrics":{"Frozen":"false","Pre-Training Dataset":"Audioset (Video+Audio)","Top-1 Accuracy":"64.7"},"uses_additional_data":false},{"leaderboard":"/sota/self-supervised-action-recognition-on-hmdb51","task":"Self-Supervised Action Recognition","dataset":"HMDB51","model":"AVID (Modified R2+1D-18 on Audioset)","rank_in_archive_order":20,"of":48,"metrics":{"Frozen":"false","Pre-Training Dataset":"Audioset (Video+Audio)","Top-1 Accuracy":"64.1"},"uses_additional_data":false},{"leaderboard":"/sota/self-supervised-action-recognition-on-hmdb51","task":"Self-Supervised Action Recognition","dataset":"HMDB51","model":"AVID+CMA (Modified R2+1D-18 on Kinetics)","rank_in_archive_order":25,"of":48,"metrics":{"Frozen":"false","Pre-Training Dataset":"Kinetics400 (Video+Audio)","Top-1 Accuracy":"60.8"},"uses_additional_data":false},{"leaderboard":"/sota/self-supervised-action-recognition-on-hmdb51","task":"Self-Supervised Action Recognition","dataset":"HMDB51","model":"AVID (Modified R2+1D-18 on Kinetics)","rank_in_archive_order":27,"of":48,"metrics":{"Frozen":"false","Pre-Training Dataset":"Kinetics400 (Video+Audio)","Top-1 Accuracy":"59.9"},"uses_additional_data":false},{"leaderboard":"/sota/self-supervised-action-recognition-on-hmdb51-1","task":"Self-Supervised Action Recognition","dataset":"HMDB51 (finetuned)","model":"AVID","rank_in_archive_order":8,"of":14,"metrics":{"Top-1 Accuracy":"64.7"},"uses_additional_data":false},{"leaderboard":"/sota/self-supervised-action-recognition-on-ucf101","task":"Self-Supervised Action Recognition","dataset":"UCF101","model":"AVID+CMA (Modified R2+1D-18 on Audioset)","rank_in_archive_order":18,"of":53,"metrics":{"3-fold Accuracy":"91.5","Frozen":"false","Pre-Training Dataset":"Audioset (Audio+Video)"},"uses_additional_data":false},{"leaderboard":"/sota/self-supervised-action-recognition-on-ucf101","task":"Self-Supervised Action Recognition","dataset":"UCF101","model":"AVID (Modified R2+1D-18 on Audioset)","rank_in_archive_order":21,"of":53,"metrics":{"3-fold Accuracy":"91.0","Frozen":"false","Pre-Training Dataset":"Audioset (Audio+Video)"},"uses_additional_data":false},{"leaderboard":"/sota/self-supervised-action-recognition-on-ucf101","task":"Self-Supervised Action Recognition","dataset":"UCF101","model":"AVID+CMA (Modified R2+1D-18 on Kinetics)","rank_in_archive_order":26,"of":53,"metrics":{"3-fold Accuracy":"87.5","Frozen":"false","Pre-Training Dataset":"Kinetics400 (Audio+Video)"},"uses_additional_data":false},{"leaderboard":"/sota/self-supervised-action-recognition-on-ucf101","task":"Self-Supervised Action Recognition","dataset":"UCF101","model":"AVID (Modified R2+1D-18 on Kinetics)","rank_in_archive_order":27,"of":53,"metrics":{"3-fold Accuracy":"86.9","Frozen":"false","Pre-Training Dataset":"Kinetics400 (Audio+Video)"},"uses_additional_data":false},{"leaderboard":"/sota/self-supervised-action-recognition-on-ucf101-1","task":"Self-Supervised Action Recognition","dataset":"UCF101 (finetuned)","model":"AVID","rank_in_archive_order":7,"of":14,"metrics":{"3-fold Accuracy":"91.5"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2004.12943","mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}