{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/vicreg-variance-invariance-covariance","title":"VICReg: Variance-Invariance-Covariance Regularization for Self-Supervised Learning","arxiv_id":"2105.04906","date":"2021-05-11","proceeding":"NeurIPS 2021 12","authors":["Adrien Bardes","Jean Ponce","Yann Lecun"],"abstract":"Recent self-supervised methods for image representation learning are based on maximizing the agreement between embedding vectors from different views of the same image. A trivial solution is obtained when the encoder outputs constant vectors. This collapse problem is often avoided through implicit biases in the learning architecture, that often lack a clear justification or interpretation. In this paper, we introduce VICReg (Variance-Invariance-Covariance Regularization), a method that explicitly avoids the collapse problem with a simple regularization term on the variance of the embeddings along each dimension individually. VICReg combines the variance term with a decorrelation mechanism based on redundancy reduction and covariance regularization, and achieves results on par with the state of the art on several downstream tasks. In addition, we show that incorporating our new variance term into other methods helps stabilize the training and leads to performance improvements.","url_abs":"https://arxiv.org/abs/2105.04906v3","url_pdf":"https://arxiv.org/pdf/2105.04906v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"vicreg-variance-invariance-covariance","repo_url":"https://github.com/facebookresearch/vicreg","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"vicreg-variance-invariance-covariance","repo_url":"https://github.com/AnnaManasyan/VICReg","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"vicreg-variance-invariance-covariance","repo_url":"https://github.com/FloCF/SSL_pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"vicreg-variance-invariance-covariance","repo_url":"https://github.com/lightly-ai/lightly","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"vicreg-variance-invariance-covariance","repo_url":"https://github.com/vturrisi/solo-learn","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"vicreg-variance-invariance-covariance","repo_url":"https://gitlab.com/birder/birder","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"representation-learning","task_name":"Representation Learning"},{"task_slug":"self-supervised-image-classification","task_name":"Self-Supervised Image Classification"},{"task_slug":"self-supervised-learning","task_name":"Self-Supervised Learning"},{"task_slug":"semi-supervised-image-classification","task_name":"Semi-Supervised Image Classification"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/self-supervised-image-classification-on","task":"Self-Supervised Image Classification","dataset":"ImageNet","model":"VICReg (ResNet50)","rank_in_archive_order":90,"of":144,"metrics":{"Number of Params":"24M","Top 1 Accuracy":"73.2","Top 5 Accuracy":"91.1"},"uses_additional_data":false},{"leaderboard":"/sota/semi-supervised-image-classification-on-1","task":"Semi-Supervised Image Classification","dataset":"ImageNet - 1% labeled data","model":"VICREG (Resnet-50)","rank_in_archive_order":47,"of":65,"metrics":{"Top 1 Accuracy":"54.8%","Top 5 Accuracy":"79.4%"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2105.04906","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.04906"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/lightly-ai/lightly","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/FloCF/SSL_pytorch","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/vturrisi/solo-learn","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/vicreg","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://gitlab.com/birder/birder","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/AnnaManasyan/VICReg","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":4,"ran_draft_wrong":3,"ran_fixture":2,"ran_honours":1,"ran_violates":1,"unverified":6},"by_repo_kind":{"official":{"samples":5,"ran":4,"repositories":1},"listed":{"samples":11,"ran":6,"repositories":3}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":1,"samples":[{"code_sha256_prefix":"90bc3c3bb8e6523b","entry":"FullGatherLayer","repo":"facebookresearch/vicreg","repo_kind":"official","path":"main_vicreg.py","file_url":"https://github.com/facebookresearch/vicreg/blob/HEAD/main_vicreg.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"90bc3c3bb8e6523b"}},{"code_sha256_prefix":"4073146360f534c5","entry":"GatherLayer","repo":"lightly-ai/lightly","repo_kind":"listed","path":"lightly/loss/vicreg_loss.py","file_url":"https://github.com/lightly-ai/lightly/blob/HEAD/lightly/loss/vicreg_loss.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4073146360f534c5"}},{"code_sha256_prefix":"19322d1e968e9137","entry":"Projector","repo":"facebookresearch/vicreg","repo_kind":"official","path":"main_vicreg.py","file_url":"https://github.com/facebookresearch/vicreg/blob/HEAD/main_vicreg.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"19322d1e968e9137"}},{"code_sha256_prefix":"c796daf86600fb9e","entry":"VICReg","repo":"FloCF/SSL_pytorch","repo_kind":"listed","path":"torchselfsup/models/vicreg.py","file_url":"https://github.com/FloCF/SSL_pytorch/blob/HEAD/torchselfsup/models/vicreg.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c796daf86600fb9e"}},{"code_sha256_prefix":"23452a5a035b134a","entry":"VICRegLoss","repo":"lightly-ai/lightly","repo_kind":"listed","path":"lightly/loss/vicreg_loss.py","file_url":"https://github.com/lightly-ai/lightly/blob/HEAD/lightly/loss/vicreg_loss.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"23452a5a035b134a"}},{"code_sha256_prefix":"b0f936d4d6ae3b8c","entry":"accuracy","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"b0f936d4d6ae3b8c"}},{"code_sha256_prefix":"0a59c9ccf5135868","entry":"adjust_learning_rate","repo":"facebookresearch/vicreg","repo_kind":"official","path":"main_vicreg.py","file_url":"https://github.com/facebookresearch/vicreg/blob/HEAD/main_vicreg.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0a59c9ccf5135868"}},{"code_sha256_prefix":"28c640db8f721c0d","entry":"exclude_bias_and_norm","repo":"facebookresearch/vicreg","repo_kind":"official","path":"main_vicreg.py","file_url":"https://github.com/facebookresearch/vicreg/blob/HEAD/main_vicreg.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"28c640db8f721c0d"}},{"code_sha256_prefix":"48e54f52f61ab9b8","entry":"invariance_loss","repo":"lightly-ai/lightly","repo_kind":"listed","path":"lightly/loss/vicreg_loss.py","file_url":"https://github.com/lightly-ai/lightly/blob/HEAD/lightly/loss/vicreg_loss.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"48e54f52f61ab9b8"}},{"code_sha256_prefix":"3e30d88eaef01190","entry":"off_diagonal","repo":"AnnaManasyan/VICReg","repo_kind":"listed","path":"loss.py","file_url":"https://github.com/AnnaManasyan/VICReg/blob/HEAD/loss.py","link_basis":"plan_row","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3e30d88eaef01190"}},{"code_sha256_prefix":"adb0bbb17650f6fe","entry":"variance_loss","repo":"lightly-ai/lightly","repo_kind":"listed","path":"lightly/loss/vicreg_loss.py","file_url":"https://github.com/lightly-ai/lightly/blob/HEAD/lightly/loss/vicreg_loss.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"adb0bbb17650f6fe"}},{"code_sha256_prefix":"a8066cb8d5d3e19d","entry":"VICReg","repo":"facebookresearch/vicreg","repo_kind":"official","path":"main_vicreg.py","file_url":"https://github.com/facebookresearch/vicreg/blob/HEAD/main_vicreg.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a8066cb8d5d3e19d"}},{"code_sha256_prefix":"73fbb91c4b92324b","entry":"augment","repo":"AnnaManasyan/VICReg","repo_kind":"listed","path":"utils.py","file_url":"https://github.com/AnnaManasyan/VICReg/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"73fbb91c4b92324b"}},{"code_sha256_prefix":"0c5a6989161b85e9","entry":"cov_loss","repo":"AnnaManasyan/VICReg","repo_kind":"listed","path":"loss.py","file_url":"https://github.com/AnnaManasyan/VICReg/blob/HEAD/loss.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0c5a6989161b85e9"}},{"code_sha256_prefix":"f4b990248d9c5170","entry":"gather","repo":"lightly-ai/lightly","repo_kind":"listed","path":"lightly/loss/vicreg_loss.py","file_url":"https://github.com/lightly-ai/lightly/blob/HEAD/lightly/loss/vicreg_loss.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f4b990248d9c5170"}},{"code_sha256_prefix":"1eeada3f42590ae7","entry":"optim","repo":"AnnaManasyan/VICReg","repo_kind":"listed","path":"utils.py","file_url":"https://github.com/AnnaManasyan/VICReg/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1eeada3f42590ae7"}},{"code_sha256_prefix":"62a1a07b187aa7a9","entry":"std_loss","repo":"AnnaManasyan/VICReg","repo_kind":"listed","path":"loss.py","file_url":"https://github.com/AnnaManasyan/VICReg/blob/HEAD/loss.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"62a1a07b187aa7a9"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}