{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/better-plain-vit-baselines-for-imagenet-1k","title":"Better plain ViT baselines for ImageNet-1k","arxiv_id":"2205.01580","date":"2022-05-03","proceeding":null,"authors":["Lucas Beyer","Xiaohua Zhai","Alexander Kolesnikov"],"abstract":"It is commonly accepted that the Vision Transformer model requires sophisticated regularization techniques to excel at ImageNet-1k scale data. Surprisingly, we find this is not the case and standard data augmentation is sufficient. This note presents a few minor modifications to the original Vision Transformer (ViT) vanilla training setting that dramatically improve the performance of plain ViT models. Notably, 90 epochs of training surpass 76% top-1 accuracy in under seven hours on a TPUv3-8, similar to the classic ResNet50 baseline, and 300 epochs of training reach 80% in less than one day.","url_abs":"https://arxiv.org/abs/2205.01580v1","url_pdf":"https://arxiv.org/pdf/2205.01580v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"better-plain-vit-baselines-for-imagenet-1k","repo_url":"https://github.com/google-research/big_vision","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"jax","reach":null},{"paper_slug":"better-plain-vit-baselines-for-imagenet-1k","repo_url":"https://github.com/conceptofmind/Simple-ViT-flax","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":null},{"paper_slug":"better-plain-vit-baselines-for-imagenet-1k","repo_url":"https://github.com/jeongsoop/rgb-no-more","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"better-plain-vit-baselines-for-imagenet-1k","repo_url":"https://github.com/osiriszjq/impulse_init","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"better-plain-vit-baselines-for-imagenet-1k","repo_url":"https://github.com/yuyangshu/retinavit","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"better-plain-vit-baselines-for-imagenet-1k","repo_url":"https://github.com/pwc-1/Paper-9/tree/main/5/vit","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null},{"paper_slug":"better-plain-vit-baselines-for-imagenet-1k","repo_url":"https://gitlab.com/birder/birder","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"data-augmentation","task_name":"Data Augmentation"},{"task_slug":"image-classification","task_name":"Image Classification"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"},{"method_slug":"vision-transformer","method_name":"Vision Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2205.01580","atlas_url":"https://app.syntology.ai/?focus=2205.01580","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2205.01580"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/google-research/big_vision","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jeongsoop/rgb-no-more","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/conceptofmind/Simple-ViT-flax","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yuyangshu/retinavit","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/pwc-1/Paper-9/tree/main/5/vit","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://gitlab.com/birder/birder","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/osiriszjq/impulse_init","reach":null}],"summary":{"ran":14,"ran_honours":5,"ran_fixture":2,"ran_violates":1,"ran_draft_wrong":1,"unverified":1},"by_repo_kind":{"listed":{"samples":20,"ran":19,"repositories":3}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":4,"samples":[{"code_sha256_prefix":"0a92f4de53a489b4","entry":"Model","repo":"yuyangshu/retinavit","repo_kind":"listed","path":"big_vision/models/mlp_mixer.py","file_url":"https://github.com/yuyangshu/retinavit/blob/HEAD/big_vision/models/mlp_mixer.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0a92f4de53a489b4"}},{"code_sha256_prefix":"0440e95034d883bd","entry":"SpatialConv2d_Linear_init","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"0440e95034d883bd"}},{"code_sha256_prefix":"b04417d600247aaa","entry":"SpatialConv2d_init","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"b04417d600247aaa"}},{"code_sha256_prefix":"2c4ed3f75ab6ee14","entry":"dot_product_attention","repo":"yuyangshu/retinavit","repo_kind":"listed","path":"big_vision/models/attn_override.py","file_url":"https://github.com/yuyangshu/retinavit/blob/HEAD/big_vision/models/attn_override.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2c4ed3f75ab6ee14"}},{"code_sha256_prefix":"c8927d5d53a840b6","entry":"dot_product_attention_weights","repo":"yuyangshu/retinavit","repo_kind":"listed","path":"big_vision/models/attn_override.py","file_url":"https://github.com/yuyangshu/retinavit/blob/HEAD/big_vision/models/attn_override.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c8927d5d53a840b6"}},{"code_sha256_prefix":"75f9a36fc8351778","entry":"fix_old_checkpoints","repo":"yuyangshu/retinavit","repo_kind":"listed","path":"big_vision/models/bit.py","file_url":"https://github.com/yuyangshu/retinavit/blob/HEAD/big_vision/models/bit.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"75f9a36fc8351778"}},{"code_sha256_prefix":"f40ba1fc76a56621","entry":"get","repo":"yuyangshu/retinavit","repo_kind":"listed","path":"big_vision/datasets/core.py","file_url":"https://github.com/yuyangshu/retinavit/blob/HEAD/big_vision/datasets/core.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f40ba1fc76a56621"}},{"code_sha256_prefix":"4f697aeaa3747cc2","entry":"get_block_desc","repo":"yuyangshu/retinavit","repo_kind":"listed","path":"big_vision/models/bit.py","file_url":"https://github.com/yuyangshu/retinavit/blob/HEAD/big_vision/models/bit.py","link_basis":"plan_row","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4f697aeaa3747cc2"}},{"code_sha256_prefix":"5c44a598f86dadfc","entry":"get_count","repo":"yuyangshu/retinavit","repo_kind":"listed","path":"big_vision/optax.py","file_url":"https://github.com/yuyangshu/retinavit/blob/HEAD/big_vision/optax.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5c44a598f86dadfc"}},{"code_sha256_prefix":"ec1bf067066bb434","entry":"get_posemb","repo":"yuyangshu/retinavit","repo_kind":"listed","path":"big_vision/models/vit.py","file_url":"https://github.com/yuyangshu/retinavit/blob/HEAD/big_vision/models/vit.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ec1bf067066bb434"}},{"code_sha256_prefix":"ce9f577538e842e0","entry":"impulse_init","repo":"osiriszjq/impulse_init","repo_kind":"listed","path":"vit.py","file_url":"https://github.com/osiriszjq/impulse_init/blob/HEAD/vit.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ce9f577538e842e0"}},{"code_sha256_prefix":"9d78a212d314b83e","entry":"magnitude","repo":"yuyangshu/retinavit","repo_kind":"listed","path":"big_vision/models/attn_override.py","file_url":"https://github.com/yuyangshu/retinavit/blob/HEAD/big_vision/models/attn_override.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9d78a212d314b83e"}},{"code_sha256_prefix":"a0fec96972fe8326","entry":"make_for_inference","repo":"yuyangshu/retinavit","repo_kind":"listed","path":"big_vision/input_pipeline.py","file_url":"https://github.com/yuyangshu/retinavit/blob/HEAD/big_vision/input_pipeline.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a0fec96972fe8326"}},{"code_sha256_prefix":"2b3549774154825a","entry":"npload","repo":"yuyangshu/retinavit","repo_kind":"listed","path":"big_vision/utils.py","file_url":"https://github.com/yuyangshu/retinavit/blob/HEAD/big_vision/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2b3549774154825a"}},{"code_sha256_prefix":"7b17682564107576","entry":"onehot","repo":"yuyangshu/retinavit","repo_kind":"listed","path":"big_vision/utils.py","file_url":"https://github.com/yuyangshu/retinavit/blob/HEAD/big_vision/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7b17682564107576"}},{"code_sha256_prefix":"5ad1f39431340fbb","entry":"pad_shard_unpad","repo":"yuyangshu/retinavit","repo_kind":"listed","path":"big_vision/utils.py","file_url":"https://github.com/yuyangshu/retinavit/blob/HEAD/big_vision/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5ad1f39431340fbb"}},{"code_sha256_prefix":"6ba8cee9f5daea41","entry":"pair","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"6ba8cee9f5daea41"}},{"code_sha256_prefix":"4f5dd3aec9da6ede","entry":"posemb_conv","repo":"yuyangshu/retinavit","repo_kind":"listed","path":"big_vision/models/vit.py","file_url":"https://github.com/yuyangshu/retinavit/blob/HEAD/big_vision/models/vit.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4f5dd3aec9da6ede"}},{"code_sha256_prefix":"cfe18b9cc8afbfe9","entry":"posemb_sincos_2d","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"cfe18b9cc8afbfe9"}},{"code_sha256_prefix":"b848f8a0a583e58b","entry":"posemb_sincos_2d","repo":"conceptofmind/Simple-ViT-flax","repo_kind":"listed","path":"simple_vit.py","file_url":"https://github.com/conceptofmind/Simple-ViT-flax/blob/HEAD/simple_vit.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b848f8a0a583e58b"}},{"code_sha256_prefix":"2d67a8a756fe7cf8","entry":"posemb_sincos_2d","repo":"yuyangshu/retinavit","repo_kind":"listed","path":"big_vision/models/vit.py","file_url":"https://github.com/yuyangshu/retinavit/blob/HEAD/big_vision/models/vit.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2d67a8a756fe7cf8"}},{"code_sha256_prefix":"c272f024d815d59d","entry":"standardize","repo":"yuyangshu/retinavit","repo_kind":"listed","path":"big_vision/models/bit_paper.py","file_url":"https://github.com/yuyangshu/retinavit/blob/HEAD/big_vision/models/bit_paper.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c272f024d815d59d"}},{"code_sha256_prefix":"37b141b05f87de19","entry":"weight_standardize","repo":"yuyangshu/retinavit","repo_kind":"listed","path":"big_vision/models/bit.py","file_url":"https://github.com/yuyangshu/retinavit/blob/HEAD/big_vision/models/bit.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"37b141b05f87de19"}},{"code_sha256_prefix":"1f64c95d3b75c46b","entry":"find_states","repo":"yuyangshu/retinavit","repo_kind":"listed","path":"big_vision/optax.py","file_url":"https://github.com/yuyangshu/retinavit/blob/HEAD/big_vision/optax.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1f64c95d3b75c46b"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}