{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/learning-audio-visual-representations-with","title":"Active Contrastive Learning of Audio-Visual Video Representations","arxiv_id":"2009.09805","date":"2020-08-31","proceeding":"ICLR 2021 1","authors":["Shuang Ma","Zhaoyang Zeng","Daniel McDuff","Yale Song"],"abstract":"Contrastive learning has been shown to produce generalizable representations of audio and visual data by maximizing the lower bound on the mutual information (MI) between different views of an instance. However, obtaining a tight lower bound requires a sample size exponential in MI and thus a large set of negative samples. We can incorporate more samples by building a large queue-based dictionary, but there are theoretical limits to performance improvements even with a large number of negative samples. We hypothesize that \\textit{random negative sampling} leads to a highly redundant dictionary that results in suboptimal representations for downstream tasks. In this paper, we propose an active contrastive learning approach that builds an \\textit{actively sampled} dictionary with diverse and informative items, which improves the quality of negative samples and improves performances on tasks where there is high mutual information in the data, e.g., video classification. Our model achieves state-of-the-art performance on challenging audio and visual downstream benchmarks including UCF101, HMDB51 and ESC50.\\footnote{Code is available at: \\url{https://github.com/yunyikristy/CM-ACC}}","url_abs":"https://arxiv.org/abs/2009.09805v2","url_pdf":"https://arxiv.org/pdf/2009.09805v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"learning-audio-visual-representations-with","repo_url":"https://github.com/yunyikristy/CM-ACC","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"contrastive-learning","task_name":"Contrastive Learning"},{"task_slug":"representation-learning","task_name":"Representation Learning"},{"task_slug":"video-classification","task_name":"Video Classification"}],"methods":[{"method_slug":"contrastive-learning","method_name":"Contrastive Learning"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2009.09805","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2009.09805"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yunyikristy/CM-ACC","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_honours":1,"unverified":9},"by_repo_kind":{"official":{"samples":10,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"1df9a5ffd9b38c34","entry":"pil_loader","repo":"yunyikristy/CM-ACC","repo_kind":"official","path":"datasets/activitynet.py","file_url":"https://github.com/yunyikristy/CM-ACC/blob/HEAD/datasets/activitynet.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1df9a5ffd9b38c34"}},{"code_sha256_prefix":"aae0a4f3535e8f26","entry":"accimage_loader","repo":"yunyikristy/CM-ACC","repo_kind":"official","path":"datasets/activitynet.py","file_url":"https://github.com/yunyikristy/CM-ACC/blob/HEAD/datasets/activitynet.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"aae0a4f3535e8f26"}},{"code_sha256_prefix":"3c68e0b906dbc9ef","entry":"conv3x3","repo":"yunyikristy/CM-ACC","repo_kind":"official","path":"resnet1d.py","file_url":"https://github.com/yunyikristy/CM-ACC/blob/HEAD/resnet1d.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3c68e0b906dbc9ef"}},{"code_sha256_prefix":"b1b989baab3936b2","entry":"distance","repo":"yunyikristy/CM-ACC","repo_kind":"official","path":"kmeanspp.py","file_url":"https://github.com/yunyikristy/CM-ACC/blob/HEAD/kmeanspp.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b1b989baab3936b2"}},{"code_sha256_prefix":"671f375fc050046b","entry":"get_centroids","repo":"yunyikristy/CM-ACC","repo_kind":"official","path":"kmeanspp.py","file_url":"https://github.com/yunyikristy/CM-ACC/blob/HEAD/kmeanspp.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"671f375fc050046b"}},{"code_sha256_prefix":"b934dfd4a7bd8467","entry":"get_fine_tuning_parameters","repo":"yunyikristy/CM-ACC","repo_kind":"official","path":"models/densenet.py","file_url":"https://github.com/yunyikristy/CM-ACC/blob/HEAD/models/densenet.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b934dfd4a7bd8467"}},{"code_sha256_prefix":"e83fbcc5be03a1f6","entry":"get_mean","repo":"yunyikristy/CM-ACC","repo_kind":"official","path":"mean.py","file_url":"https://github.com/yunyikristy/CM-ACC/blob/HEAD/mean.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e83fbcc5be03a1f6"}},{"code_sha256_prefix":"9533d248536bab4a","entry":"get_std","repo":"yunyikristy/CM-ACC","repo_kind":"official","path":"mean.py","file_url":"https://github.com/yunyikristy/CM-ACC/blob/HEAD/mean.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9533d248536bab4a"}},{"code_sha256_prefix":"f4f584599385bac3","entry":"nearest","repo":"yunyikristy/CM-ACC","repo_kind":"official","path":"kmeanspp.py","file_url":"https://github.com/yunyikristy/CM-ACC/blob/HEAD/kmeanspp.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f4f584599385bac3"}},{"code_sha256_prefix":"969914385b6159ef","entry":"video_loader","repo":"yunyikristy/CM-ACC","repo_kind":"official","path":"datasets/activitynet.py","file_url":"https://github.com/yunyikristy/CM-ACC/blob/HEAD/datasets/activitynet.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"969914385b6159ef"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}