{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/an-empirical-study-of-training-self","title":"An Empirical Study of Training Self-Supervised Vision Transformers","arxiv_id":"2104.02057","date":"2021-04-05","proceeding":"ICCV 2021 10","authors":["Xinlei Chen","Saining Xie","Kaiming He"],"abstract":"This paper does not describe a novel method. Instead, it studies a straightforward, incremental, yet must-know baseline given the recent progress in computer vision: self-supervised learning for Vision Transformers (ViT). While the training recipes for standard convolutional networks have been highly mature and robust, the recipes for ViT are yet to be built, especially in the self-supervised scenarios where training becomes more challenging. In this work, we go back to basics and investigate the effects of several fundamental components for training self-supervised ViT. We observe that instability is a major issue that degrades accuracy, and it can be hidden by apparently good results. We reveal that these results are indeed partial failure, and they can be improved when training is made more stable. We benchmark ViT results in MoCo v3 and several other self-supervised frameworks, with ablations in various aspects. We discuss the currently positive evidence as well as challenges and open questions. We hope that this work will provide useful data points and experience for future research.","url_abs":"https://arxiv.org/abs/2104.02057v4","url_pdf":"https://arxiv.org/pdf/2104.02057v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"an-empirical-study-of-training-self","repo_url":"https://github.com/facebookresearch/moco-v3","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"an-empirical-study-of-training-self","repo_url":"https://github.com/CupidJay/MoCov3-pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"an-empirical-study-of-training-self","repo_url":"https://github.com/KAIST-AILab/MaskedAutoencoder-Jax","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"an-empirical-study-of-training-self","repo_url":"https://github.com/Westlake-AI/openmixup","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"an-empirical-study-of-training-self","repo_url":"https://github.com/oliverrensu/sdmp","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"an-empirical-study-of-training-self","repo_url":"https://github.com/oneflow-inc/libai","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"an-empirical-study-of-training-self","repo_url":"https://github.com/open-mmlab/mmselfsup","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"an-empirical-study-of-training-self","repo_url":"https://github.com/xiyue-wang/transpath","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"GPL-3.0"}},{"paper_slug":"an-empirical-study-of-training-self","repo_url":"https://github.com/toliz/moco-v3","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"out-of-distribution-generalization","task_name":"Out-of-Distribution Generalization"},{"task_slug":"self-supervised-image-classification","task_name":"Self-Supervised Image Classification"},{"task_slug":"self-supervised-learning","task_name":"Self-Supervised Learning"}],"methods":[{"method_slug":"batch-normalization","method_name":"Batch Normalization"},{"method_slug":"infonce","method_name":"InfoNCE"},{"method_slug":"moco","method_name":"MoCo"},{"method_slug":"moco-v3","method_name":"MoCo v3"}],"datasets_introduced":[],"methods_introduced":[{"slug":"moco-v3","name":"MoCo v3","full_name":"MoCo v3"}],"results":[{"leaderboard":"/sota/self-supervised-image-classification-on","task":"Self-Supervised Image Classification","dataset":"ImageNet","model":"MoCo v3 (ViT-BN-L/7)","rank_in_archive_order":18,"of":144,"metrics":{"Number of Params":"304M","Top 1 Accuracy":"81.0%"},"uses_additional_data":false},{"leaderboard":"/sota/self-supervised-image-classification-on","task":"Self-Supervised Image Classification","dataset":"ImageNet","model":"MoCo v3 (ViT-BN-H)","rank_in_archive_order":37,"of":144,"metrics":{"Number of Params":"700M","Top 1 Accuracy":"79.1%"},"uses_additional_data":false},{"leaderboard":"/sota/self-supervised-image-classification-on","task":"Self-Supervised Image Classification","dataset":"ImageNet","model":"MoCo v3 (ViT-H)","rank_in_archive_order":47,"of":144,"metrics":{"Number of Params":"632M","Top 1 Accuracy":"78.1%"},"uses_additional_data":false},{"leaderboard":"/sota/self-supervised-image-classification-on","task":"Self-Supervised Image Classification","dataset":"ImageNet","model":"MoCo v3 (ViT-L)","rank_in_archive_order":51,"of":144,"metrics":{"Number of Params":"307M","Top 1 Accuracy":"77.6%"},"uses_additional_data":false},{"leaderboard":"/sota/self-supervised-image-classification-on","task":"Self-Supervised Image Classification","dataset":"ImageNet","model":"MoCo v3 (ViT-B/16)","rank_in_archive_order":58,"of":144,"metrics":{"Number of Params":"86M","Top 1 Accuracy":"76.7%"},"uses_additional_data":false},{"leaderboard":"/sota/self-supervised-image-classification-on-1","task":"Self-Supervised Image Classification","dataset":"ImageNet (finetuned)","model":"MoCo v3 (ViT-L/16)","rank_in_archive_order":35,"of":65,"metrics":{"Number of Params":"304M","Top 1 Accuracy":"84.1%"},"uses_additional_data":false},{"leaderboard":"/sota/self-supervised-image-classification-on-1","task":"Self-Supervised Image Classification","dataset":"ImageNet (finetuned)","model":"MoCo v3 (ViT-B/16)","rank_in_archive_order":44,"of":65,"metrics":{"Number of Params":"86M","Top 1 Accuracy":"83.2%"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2104.02057","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.02057"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Westlake-AI/openmixup","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/oliverrensu/sdmp","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/toliz/moco-v3","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/xiyue-wang/transpath","reach":{"status":"ok","spdx":"GPL-3.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/oneflow-inc/libai","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/KAIST-AILab/MaskedAutoencoder-Jax","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/moco-v3","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/open-mmlab/mmselfsup","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/CupidJay/MoCov3-pytorch","reach":null}],"summary":{"ran_fixture":1,"ran_draft_wrong":1},"by_repo_kind":{"listed":{"samples":1,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":2,"samples":[{"code_sha256_prefix":"983576fa67f4f01e","entry":"accuracy","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"983576fa67f4f01e"}},{"code_sha256_prefix":"c73266677f2758b5","entry":"ctr","repo":"CupidJay/MoCov3-pytorch","repo_kind":"listed","path":"main_moco_pretraining_v3.py","file_url":"https://github.com/CupidJay/MoCov3-pytorch/blob/HEAD/main_moco_pretraining_v3.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c73266677f2758b5"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}