{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/going-deeper-with-image-transformers","title":"Going deeper with Image Transformers","arxiv_id":"2103.17239","date":"2021-03-31","proceeding":"ICCV 2021 10","authors":["Hugo Touvron","Matthieu Cord","Alexandre Sablayrolles","Gabriel Synnaeve","Hervé Jégou"],"abstract":"Transformers have been recently adapted for large scale image classification, achieving high scores shaking up the long supremacy of convolutional neural networks. However the optimization of image transformers has been little studied so far. In this work, we build and optimize deeper transformer networks for image classification. In particular, we investigate the interplay of architecture and optimization of such dedicated transformers. We make two transformers architecture changes that significantly improve the accuracy of deep transformers. This leads us to produce models whose performance does not saturate early with more depth, for instance we obtain 86.5% top-1 accuracy on Imagenet when training with no external data, we thus attain the current SOTA with less FLOPs and parameters. Moreover, our best model establishes the new state of the art on Imagenet with Reassessed labels and Imagenet-V2 / match frequency, in the setting with no additional training data. We share our code and models.","url_abs":"https://arxiv.org/abs/2103.17239v2","url_pdf":"https://arxiv.org/pdf/2103.17239v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"going-deeper-with-image-transformers","repo_url":"https://github.com/rwightman/pytorch-image-models","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"going-deeper-with-image-transformers","repo_url":"https://github.com/facebookresearch/deit","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":null},{"paper_slug":"going-deeper-with-image-transformers","repo_url":"https://github.com/CPones/Paddle_CaiT","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"paddle","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"going-deeper-with-image-transformers","repo_url":"https://github.com/DarshanDeshpande/jax-models","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":null},{"paper_slug":"going-deeper-with-image-transformers","repo_url":"https://github.com/IMvision12/keras-vision-models","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"going-deeper-with-image-transformers","repo_url":"https://github.com/ahmedelmahy/myownvit","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"going-deeper-with-image-transformers","repo_url":"https://github.com/conceptofmind/CaiT-Flax","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":null},{"paper_slug":"going-deeper-with-image-transformers","repo_url":"https://github.com/holdfire/CLS","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"going-deeper-with-image-transformers","repo_url":"https://github.com/holdfire/FAS","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"going-deeper-with-image-transformers","repo_url":"https://github.com/jaketae/res-mlp","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"going-deeper-with-image-transformers","repo_url":"https://github.com/liuxingwt/CLS","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"going-deeper-with-image-transformers","repo_url":"https://github.com/lucidrains/vit-pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"going-deeper-with-image-transformers","repo_url":"https://github.com/martinsbruveris/tensorflow-image-models","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":null},{"paper_slug":"going-deeper-with-image-transformers","repo_url":"https://github.com/sayakpaul/cait-tf","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"going-deeper-with-image-transformers","repo_url":"https://github.com/ttt496/vit-pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"going-deeper-with-image-transformers","repo_url":"https://github.com/2023-MindSpore-4/Code8/tree/main/cait","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null},{"paper_slug":"going-deeper-with-image-transformers","repo_url":"https://github.com/2024-MindSpore-1/Code7/tree/main/cait","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null},{"paper_slug":"going-deeper-with-image-transformers","repo_url":"https://github.com/BR-IDL/PaddleViT/blob/main/image_classification/CaiT","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"paddle","reach":null},{"paper_slug":"going-deeper-with-image-transformers","repo_url":"https://github.com/Mind23-2/MindCode-4","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":{"status":"ok"}},{"paper_slug":"going-deeper-with-image-transformers","repo_url":"https://github.com/MindSpore-paper-code-3/code6/tree/main/cait","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null},{"paper_slug":"going-deeper-with-image-transformers","repo_url":"https://gitlab.com/birder/birder","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"image-classification","task_name":"Image Classification"},{"task_slug":"transfer-learning","task_name":"Transfer Learning"},{"task_slug":"image-classification","task_name":"image-classification"}],"methods":[{"method_slug":"attention","method_name":"Attention"},{"method_slug":"cait","method_name":"CaiT"},{"method_slug":"ca","method_name":"Class Attention"},{"method_slug":"deit","method_name":"DeiT"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"feedforward-network","method_name":"Feedforward Network"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"layerscale","method_name":"LayerScale"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"}],"datasets_introduced":[],"methods_introduced":[{"slug":"ca","name":"Class Attention","full_name":"Class Attention"},{"slug":"layerscale","name":"LayerScale","full_name":"LayerScale"}],"results":[{"leaderboard":"/sota/image-classification-on-cifar-10","task":"Image Classification","dataset":"CIFAR-10","model":"CaiT-M-36 U 224","rank_in_archive_order":5,"of":265,"metrics":{"Percentage correct":"99.4"},"uses_additional_data":true},{"leaderboard":"/sota/image-classification-on-cifar-100","task":"Image Classification","dataset":"CIFAR-100","model":"CaiT-M-36 U 224","rank_in_archive_order":10,"of":211,"metrics":{"Percentage correct":"93.1"},"uses_additional_data":true},{"leaderboard":"/sota/image-classification-on-flowers-102","task":"Image Classification","dataset":"Flowers-102","model":"CaiT-M-36 U 224","rank_in_archive_order":16,"of":52,"metrics":{"Accuracy":"99.1"},"uses_additional_data":true},{"leaderboard":"/sota/image-classification-on-imagenet","task":"Image Classification","dataset":"ImageNet","model":"CaiT-M-48-448","rank_in_archive_order":140,"of":1060,"metrics":{"GFLOPs":"377.3","Number of params":"438M","Top 1 Accuracy":"86.5%"},"uses_additional_data":false},{"leaderboard":"/sota/image-classification-on-imagenet","task":"Image Classification","dataset":"ImageNet","model":"CAIT-M36-448","rank_in_archive_order":162,"of":1060,"metrics":{"GFLOPs":"247.8","Number of params":"271M","Top 1 Accuracy":"86.3%"},"uses_additional_data":false},{"leaderboard":"/sota/image-classification-on-imagenet","task":"Image Classification","dataset":"ImageNet","model":"CAIT-M-36","rank_in_archive_order":174,"of":1060,"metrics":{"GFLOPs":"173.3","Number of params":"270.9M","Top 1 Accuracy":"86.1%"},"uses_additional_data":false},{"leaderboard":"/sota/image-classification-on-imagenet","task":"Image Classification","dataset":"ImageNet","model":"CAIT-M-24","rank_in_archive_order":196,"of":1060,"metrics":{"GFLOPs":"116.1","Number of params":"185.9M","Top 1 Accuracy":"85.8%"},"uses_additional_data":false},{"leaderboard":"/sota/image-classification-on-imagenet","task":"Image Classification","dataset":"ImageNet","model":"CAIT-S-36","rank_in_archive_order":234,"of":1060,"metrics":{"GFLOPs":"48","Number of params":"68.2M","Top 1 Accuracy":"85.4%"},"uses_additional_data":false},{"leaderboard":"/sota/image-classification-on-imagenet","task":"Image Classification","dataset":"ImageNet","model":"CAIT-S-48","rank_in_archive_order":241,"of":1060,"metrics":{"GFLOPs":"63.8","Number of params":"89.5M","Top 1 Accuracy":"85.3%"},"uses_additional_data":false},{"leaderboard":"/sota/image-classification-on-imagenet","task":"Image Classification","dataset":"ImageNet","model":"CAIT-S-24","rank_in_archive_order":257,"of":1060,"metrics":{"GFLOPs":"32.2","Number of params":"46.9M","Top 1 Accuracy":"85.1%"},"uses_additional_data":false},{"leaderboard":"/sota/image-classification-on-imagenet","task":"Image Classification","dataset":"ImageNet","model":"CAIT-XS-36","rank_in_archive_order":285,"of":1060,"metrics":{"GFLOPs":"28.8","Number of params":"38.6M","Top 1 Accuracy":"84.8%"},"uses_additional_data":false},{"leaderboard":"/sota/image-classification-on-imagenet","task":"Image Classification","dataset":"ImageNet","model":"CAIT-XS-24","rank_in_archive_order":354,"of":1060,"metrics":{"GFLOPs":"19.3","Number of params":"26.6M","Top 1 Accuracy":"84.1%"},"uses_additional_data":false},{"leaderboard":"/sota/image-classification-on-imagenet","task":"Image Classification","dataset":"ImageNet","model":"CAIT-XXS-36","rank_in_archive_order":559,"of":1060,"metrics":{"GFLOPs":"14.3","Number of params":"17.3M","Top 1 Accuracy":"82.2%"},"uses_additional_data":false},{"leaderboard":"/sota/image-classification-on-imagenet","task":"Image Classification","dataset":"ImageNet","model":"CAIT-XXS-24","rank_in_archive_order":674,"of":1060,"metrics":{"GFLOPs":"9.6","Number of params":"12M","Top 1 Accuracy":"80.9%"},"uses_additional_data":false},{"leaderboard":"/sota/image-classification-on-imagenet-real","task":"Image Classification","dataset":"ImageNet ReaL","model":"CAIT-M36-448","rank_in_archive_order":19,"of":57,"metrics":{"Accuracy":"90.2%"},"uses_additional_data":false},{"leaderboard":"/sota/image-classification-on-imagenet-v2","task":"Image Classification","dataset":"ImageNet V2","model":"CAIT-M36-448","rank_in_archive_order":16,"of":33,"metrics":{"Top 1 Accuracy":"76.7"},"uses_additional_data":false},{"leaderboard":"/sota/image-classification-on-stanford-cars","task":"Image Classification","dataset":"Stanford Cars","model":"CaiT-M-36 U 224","rank_in_archive_order":5,"of":24,"metrics":{"Accuracy":"94.2"},"uses_additional_data":false},{"leaderboard":"/sota/image-classification-on-inaturalist-2018","task":"Image Classification","dataset":"iNaturalist 2018","model":"CaiT-M-36 U 224","rank_in_archive_order":22,"of":60,"metrics":{"Top-1 Accuracy":"78%"},"uses_additional_data":true},{"leaderboard":"/sota/image-classification-on-inaturalist-2019","task":"Image Classification","dataset":"iNaturalist 2019","model":"CaiT-M-36 U 224","rank_in_archive_order":10,"of":22,"metrics":{"Top-1 Accuracy":"81.8"},"uses_additional_data":true}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2103.17239","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2103.17239"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/rwightman/pytorch-image-models","reach":null},{"provenance":"deterministic:regex_extraction","url":"https://github.com/facebookresearch/deit","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/lucidrains/vit-pytorch","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/liuxingwt/CLS","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/MindSpore-paper-code-3/code6/tree/main/cait","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/2024-MindSpore-1/Code7/tree/main/cait","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/BR-IDL/PaddleViT/blob/main/image_classification/CaiT","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ttt496/vit-pytorch","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/2023-MindSpore-4/Code8/tree/main/cait","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/martinsbruveris/tensorflow-image-models","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/IMvision12/keras-vision-models","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://gitlab.com/birder/birder","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Mind23-2/MindCode-4","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/conceptofmind/CaiT-Flax","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/DarshanDeshpande/jax-models","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/CPones/Paddle_CaiT","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/holdfire/CLS","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sayakpaul/cait-tf","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jaketae/res-mlp","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ahmedelmahy/myownvit","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/holdfire/FAS","reach":null}],"summary":{"ran_draft_wrong":2,"ran_violates":2,"ran_fixture":1,"unverified":6},"by_repo_kind":{"listed":{"samples":9,"ran":3,"repositories":4}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":2,"samples":[{"code_sha256_prefix":"7794eb689431c216","entry":"dropout_layers","repo":"conceptofmind/CaiT-Flax","repo_kind":"listed","path":"cait.py","file_url":"https://github.com/conceptofmind/CaiT-Flax/blob/HEAD/cait.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7794eb689431c216"}},{"code_sha256_prefix":"aa5486a3650902d8","entry":"exists","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"aa5486a3650902d8"}},{"code_sha256_prefix":"6ba8cee9f5daea41","entry":"pair","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"6ba8cee9f5daea41"}},{"code_sha256_prefix":"07b1c6d8d9489187","entry":"window_partition","repo":"DarshanDeshpande/jax-models","repo_kind":"listed","path":"jax_models/models/swin_transformer.py","file_url":"https://github.com/DarshanDeshpande/jax-models/blob/HEAD/jax_models/models/swin_transformer.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"07b1c6d8d9489187"}},{"code_sha256_prefix":"0859cb39e97935ba","entry":"window_reverse","repo":"DarshanDeshpande/jax-models","repo_kind":"listed","path":"jax_models/models/swin_transformer.py","file_url":"https://github.com/DarshanDeshpande/jax-models/blob/HEAD/jax_models/models/swin_transformer.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0859cb39e97935ba"}},{"code_sha256_prefix":"e218ba48c9d2c190","entry":"conv_transpose","repo":"sayakpaul/cait-tf","repo_kind":"listed","path":"utils/helpers.py","file_url":"https://github.com/sayakpaul/cait-tf/blob/HEAD/utils/helpers.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e218ba48c9d2c190"}},{"code_sha256_prefix":"5fe12143b1680470","entry":"get_args","repo":"CPones/Paddle_CaiT","repo_kind":"listed","path":"CaiT/infer.py","file_url":"https://github.com/CPones/Paddle_CaiT/blob/HEAD/CaiT/infer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5fe12143b1680470"}},{"code_sha256_prefix":"160fd4002bd2e51b","entry":"get_config","repo":"CPones/Paddle_CaiT","repo_kind":"listed","path":"CaiT/config.py","file_url":"https://github.com/CPones/Paddle_CaiT/blob/HEAD/CaiT/config.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"160fd4002bd2e51b"}},{"code_sha256_prefix":"7cdb59fd6a14bd54","entry":"get_exclude_from_weight_decay_fn","repo":"CPones/Paddle_CaiT","repo_kind":"listed","path":"CaiT/utils.py","file_url":"https://github.com/CPones/Paddle_CaiT/blob/HEAD/CaiT/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7cdb59fd6a14bd54"}},{"code_sha256_prefix":"ae83452001c08f2a","entry":"mlp","repo":"sayakpaul/cait-tf","repo_kind":"listed","path":"cait/models.py","file_url":"https://github.com/sayakpaul/cait-tf/blob/HEAD/cait/models.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ae83452001c08f2a"}},{"code_sha256_prefix":"ee77e3fcb05ad040","entry":"update_config","repo":"CPones/Paddle_CaiT","repo_kind":"listed","path":"CaiT/config.py","file_url":"https://github.com/CPones/Paddle_CaiT/blob/HEAD/CaiT/config.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ee77e3fcb05ad040"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}