{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/siamese-vision-transformers-are-scalable","title":"Siamese Vision Transformers are Scalable Audio-visual Learners","arxiv_id":"2403.19638","date":"2024-03-28","proceeding":null,"authors":["Yan-Bo Lin","Gedas Bertasius"],"abstract":"Traditional audio-visual methods rely on independent audio and visual backbones, which is costly and not scalable. In this work, we investigate using an audio-visual siamese network (AVSiam) for efficient and scalable audio-visual pretraining. Our framework uses a single shared vision transformer backbone to process audio and visual inputs, improving its parameter efficiency, reducing the GPU memory footprint, and allowing us to scale our method to larger datasets and model sizes. We pretrain our model using a contrastive audio-visual matching objective with a multi-ratio random masking scheme, which enables our model to process larger audio-visual instance batches, helpful for contrastive learning. Unlike prior audio-visual methods, our method can robustly handle audio, visual, and audio-visual inputs with a single shared ViT backbone. Furthermore, despite using the shared backbone for both modalities, AVSiam achieves competitive or even better results than prior methods on AudioSet and VGGSound for audio-visual classification and retrieval. Our code is available at https://github.com/GenjiB/AVSiam","url_abs":"https://arxiv.org/abs/2403.19638v1","url_pdf":"https://arxiv.org/pdf/2403.19638v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"siamese-vision-transformers-are-scalable","repo_url":"https://github.com/genjib/avsiam","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"contrastive-learning","task_name":"Contrastive Learning"},{"task_slug":null,"task_name":"GPU"},{"task_slug":"retrieval","task_name":"Retrieval"}],"methods":[{"method_slug":"attention","method_name":"Attention"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"siamese-network","method_name":"Siamese Network"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"vision-transformer","method_name":"Vision Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2403.19638","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.19638"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/genjib/avsiam","reach":{"status":"ok"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/GenjiB/AVSiam","reach":{"status":"ok"}}],"summary":{"ran":13,"unverified":6},"by_repo_kind":{"official":{"samples":19,"ran":13,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":19,"samples":[{"code_sha256_prefix":"7ade838e8920e111","entry":"Attention","repo":"GenjiB/AVSiam","repo_kind":"official","path":"src/models/cav_mae_base.py","file_url":"https://github.com/GenjiB/AVSiam/blob/HEAD/src/models/cav_mae_base.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7ade838e8920e111"}},{"code_sha256_prefix":"534488ebed89ff98","entry":"Block","repo":"GenjiB/AVSiam","repo_kind":"official","path":"src/models/cav_mae_base.py","file_url":"https://github.com/GenjiB/AVSiam/blob/HEAD/src/models/cav_mae_base.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"534488ebed89ff98"}},{"code_sha256_prefix":"06f4ef2afc928c74","entry":"GatherLayer","repo":"GenjiB/AVSiam","repo_kind":"official","path":"src/models/cav_mae_base.py","file_url":"https://github.com/GenjiB/AVSiam/blob/HEAD/src/models/cav_mae_base.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"06f4ef2afc928c74"}},{"code_sha256_prefix":"fc419f8d3ed996b4","entry":"distributed_concat","repo":"genjib/avsiam","repo_kind":"official","path":"src/traintest_cavmae_base.py","file_url":"https://github.com/genjib/avsiam/blob/HEAD/src/traintest_cavmae_base.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fc419f8d3ed996b4"}},{"code_sha256_prefix":"e217fb47bef3abe4","entry":"get_immediate_subdirectories","repo":"genjib/avsiam","repo_kind":"official","path":"src/retrieval.py","file_url":"https://github.com/genjib/avsiam/blob/HEAD/src/retrieval.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e217fb47bef3abe4"}},{"code_sha256_prefix":"72584682b61aedb0","entry":"get_sim_mat","repo":"genjib/avsiam","repo_kind":"official","path":"src/retrieval.py","file_url":"https://github.com/genjib/avsiam/blob/HEAD/src/retrieval.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"72584682b61aedb0"}},{"code_sha256_prefix":"04fa3bcbfec2c0c2","entry":"get_similarity","repo":"genjib/avsiam","repo_kind":"official","path":"src/retrieval.py","file_url":"https://github.com/genjib/avsiam/blob/HEAD/src/retrieval.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"04fa3bcbfec2c0c2"}},{"code_sha256_prefix":"c15efdd12442aeb4","entry":"lookup_list","repo":"genjib/avsiam","repo_kind":"official","path":"src/dataloader.py","file_url":"https://github.com/genjib/avsiam/blob/HEAD/src/dataloader.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c15efdd12442aeb4"}},{"code_sha256_prefix":"a8fb3f62e2a901d7","entry":"make_index_dict","repo":"genjib/avsiam","repo_kind":"official","path":"src/dataloader.py","file_url":"https://github.com/genjib/avsiam/blob/HEAD/src/dataloader.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a8fb3f62e2a901d7"}},{"code_sha256_prefix":"cbf115a314bcd23e","entry":"make_name_dict","repo":"genjib/avsiam","repo_kind":"official","path":"src/dataloader.py","file_url":"https://github.com/genjib/avsiam/blob/HEAD/src/dataloader.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"cbf115a314bcd23e"}},{"code_sha256_prefix":"024894a396cde124","entry":"multiple_samples_collate","repo":"genjib/avsiam","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/genjib/avsiam/blob/HEAD/src/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"024894a396cde124"}},{"code_sha256_prefix":"d75abe35d8363cd2","entry":"remove_data_parallel","repo":"genjib/avsiam","repo_kind":"official","path":"src/run_cavmae_ft_base.py","file_url":"https://github.com/genjib/avsiam/blob/HEAD/src/run_cavmae_ft_base.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d75abe35d8363cd2"}},{"code_sha256_prefix":"b862a6e8aa9d205a","entry":"wa_model","repo":"genjib/avsiam","repo_kind":"official","path":"src/run_cavmae_ft_base.py","file_url":"https://github.com/genjib/avsiam/blob/HEAD/src/run_cavmae_ft_base.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b862a6e8aa9d205a"}},{"code_sha256_prefix":"d123c3dc102fe116","entry":"CAVMAE_BASE","repo":"GenjiB/AVSiam","repo_kind":"official","path":"src/models/cav_mae_base.py","file_url":"https://github.com/GenjiB/AVSiam/blob/HEAD/src/models/cav_mae_base.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d123c3dc102fe116"}},{"code_sha256_prefix":"b700fb0a3496b255","entry":"cosine_scheduler","repo":"genjib/avsiam","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/genjib/avsiam/blob/HEAD/src/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b700fb0a3496b255"}},{"code_sha256_prefix":"ba1356e8ceb654d2","entry":"get_grad_norm_","repo":"genjib/avsiam","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/genjib/avsiam/blob/HEAD/src/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ba1356e8ceb654d2"}},{"code_sha256_prefix":"1edf032c61297c85","entry":"validate","repo":"genjib/avsiam","repo_kind":"official","path":"src/traintest_cavmae_base.py","file_url":"https://github.com/genjib/avsiam/blob/HEAD/src/traintest_cavmae_base.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1edf032c61297c85"}},{"code_sha256_prefix":"64f4a07393bf408b","entry":"validate","repo":"genjib/avsiam","repo_kind":"official","path":"src/traintest_ft_base.py","file_url":"https://github.com/genjib/avsiam/blob/HEAD/src/traintest_ft_base.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"64f4a07393bf408b"}},{"code_sha256_prefix":"2c0dfd9400b965e8","entry":"validate_mlp","repo":"genjib/avsiam","repo_kind":"official","path":"src/traintest_cavmae_base.py","file_url":"https://github.com/genjib/avsiam/blob/HEAD/src/traintest_cavmae_base.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2c0dfd9400b965e8"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}