{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/exploring-plain-vision-transformer-backbones","title":"Exploring Plain Vision Transformer Backbones for Object Detection","arxiv_id":"2203.16527","date":"2022-03-30","proceeding":null,"authors":["Yanghao Li","Hanzi Mao","Ross Girshick","Kaiming He"],"abstract":"We explore the plain, non-hierarchical Vision Transformer (ViT) as a backbone network for object detection. This design enables the original ViT architecture to be fine-tuned for object detection without needing to redesign a hierarchical backbone for pre-training. With minimal adaptations for fine-tuning, our plain-backbone detector can achieve competitive results. Surprisingly, we observe: (i) it is sufficient to build a simple feature pyramid from a single-scale feature map (without the common FPN design) and (ii) it is sufficient to use window attention (without shifting) aided with very few cross-window propagation blocks. With plain ViT backbones pre-trained as Masked Autoencoders (MAE), our detector, named ViTDet, can compete with the previous leading methods that were all based on hierarchical backbones, reaching up to 61.3 AP_box on the COCO dataset using only ImageNet-1K pre-training. We hope our study will draw attention to research on plain-backbone detectors. Code for ViTDet is available in Detectron2.","url_abs":"https://arxiv.org/abs/2203.16527v2","url_pdf":"https://arxiv.org/pdf/2203.16527v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"exploring-plain-vision-transformer-backbones","repo_url":"https://github.com/facebookresearch/detectron2/tree/main/projects/ViTDet","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":null},{"paper_slug":"exploring-plain-vision-transformer-backbones","repo_url":"https://github.com/ViTAE-Transformer/ViTDet","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"exploring-plain-vision-transformer-backbones","repo_url":"https://github.com/hula-ai/DAMA","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"exploring-plain-vision-transformer-backbones","repo_url":"https://github.com/kdexd/coco-rem","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"exploring-plain-vision-transformer-backbones","repo_url":"https://github.com/vitae-transformer/qformer","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"exploring-plain-vision-transformer-backbones","repo_url":"https://github.com/MindCode-4/code-1/tree/main/vit","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null},{"paper_slug":"exploring-plain-vision-transformer-backbones","repo_url":"https://github.com/MindCode-4/code-5/tree/main/vitdet","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null},{"paper_slug":"exploring-plain-vision-transformer-backbones","repo_url":"https://github.com/PaddlePaddle/PaddleDetection/tree/develop/configs/vitdet","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"paddle","reach":null},{"paper_slug":"exploring-plain-vision-transformer-backbones","repo_url":"https://github.com/alibaba/EasyCV","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":null},{"paper_slug":"exploring-plain-vision-transformer-backbones","repo_url":"https://github.com/pwc-1/Paper-9/tree/main/1/vitdet","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null},{"paper_slug":"exploring-plain-vision-transformer-backbones","repo_url":"https://gitlab.com/birder/birder","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"cross-domain-few-shot-object-detection","task_name":"Cross-Domain Few-Shot Object Detection"},{"task_slug":"instance-segmentation","task_name":"Instance Segmentation"},{"task_slug":"object","task_name":"Object"},{"task_slug":"object-detection","task_name":"Object Detection"},{"task_slug":"object-detection-1","task_name":"object-detection"}],"methods":[{"method_slug":"1x1-convolution","method_name":"1x1 Convolution"},{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"convolution","method_name":"Convolution"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"fpn","method_name":"FPN"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"},{"method_slug":"vision-transformer","method_name":"Vision Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/cross-domain-few-shot-object-detection-on","task":"Cross-Domain Few-Shot Object Detection","dataset":"Artaxor","model":"ViTDeT-FT","rank_in_archive_order":9,"of":16,"metrics":{" mAP":"23.4"},"uses_additional_data":false},{"leaderboard":"/sota/cross-domain-few-shot-object-detection-on-1","task":"Cross-Domain Few-Shot Object Detection","dataset":"Clipark1k","model":"ViTDeT-FT","rank_in_archive_order":7,"of":10,"metrics":{" mAP":"25.6"},"uses_additional_data":false},{"leaderboard":"/sota/cross-domain-few-shot-object-detection-on-2","task":"Cross-Domain Few-Shot Object Detection","dataset":"DIOR","model":"ViTDeT-FT","rank_in_archive_order":5,"of":15,"metrics":{"mAP":"29.4"},"uses_additional_data":false},{"leaderboard":"/sota/cross-domain-few-shot-object-detection-on-3","task":"Cross-Domain Few-Shot Object Detection","dataset":"DeepFish","model":"ViTDeT-FT","rank_in_archive_order":8,"of":10,"metrics":{"mAP":"6.5"},"uses_additional_data":false},{"leaderboard":"/sota/cross-domain-few-shot-object-detection-on-neu","task":"Cross-Domain Few-Shot Object Detection","dataset":"NEU-DET","model":"ViTDeT-FT","rank_in_archive_order":4,"of":10,"metrics":{"mAP":"15.8"},"uses_additional_data":false},{"leaderboard":"/sota/cross-domain-few-shot-object-detection-on-4","task":"Cross-Domain Few-Shot Object Detection","dataset":"UODD","model":"ViTDeT-FT","rank_in_archive_order":8,"of":16,"metrics":{"mAP":"15.8"},"uses_additional_data":false},{"leaderboard":"/sota/instance-segmentation-on-coco-minival","task":"Instance Segmentation","dataset":"COCO minival","model":"ViTDet, ViT-H Cascade (multiscale)","rank_in_archive_order":10,"of":93,"metrics":{"mask AP":"53.1"},"uses_additional_data":false},{"leaderboard":"/sota/instance-segmentation-on-coco-minival","task":"Instance Segmentation","dataset":"COCO minival","model":"ViTDet, ViT-H Cascade","rank_in_archive_order":16,"of":93,"metrics":{"mask AP":"52"},"uses_additional_data":false},{"leaderboard":"/sota/instance-segmentation-on-lvis-v1-0-val","task":"Instance Segmentation","dataset":"LVIS v1.0 val","model":"ViTDet-H","rank_in_archive_order":5,"of":25,"metrics":{"mask AP":"48.1","mask APr":"36.9"},"uses_additional_data":false},{"leaderboard":"/sota/instance-segmentation-on-lvis-v1-0-val","task":"Instance Segmentation","dataset":"LVIS v1.0 val","model":"ViTDet-L","rank_in_archive_order":6,"of":25,"metrics":{"mask AP":"46.0","mask APr":"34.3"},"uses_additional_data":false},{"leaderboard":"/sota/object-detection-on-coco-minival","task":"Object Detection","dataset":"COCO minival","model":"ViTDet, ViT-H Cascade (multiscale)","rank_in_archive_order":18,"of":220,"metrics":{"box AP":"61.3"},"uses_additional_data":false},{"leaderboard":"/sota/object-detection-on-coco-minival","task":"Object Detection","dataset":"COCO minival","model":"ViTDet, ViT-H Cascade","rank_in_archive_order":23,"of":220,"metrics":{"box AP":"60.4"},"uses_additional_data":false},{"leaderboard":"/sota/object-detection-on-coco-o","task":"Object Detection","dataset":"COCO-O","model":"ViTDet (ViT-H)","rank_in_archive_order":10,"of":45,"metrics":{"Average mAP":"34.3"},"uses_additional_data":false},{"leaderboard":"/sota/object-detection-on-coco-o","task":"Object Detection","dataset":"COCO-O","model":"ViTDet\n(ViT-H)","rank_in_archive_order":43,"of":45,"metrics":{"Effective Robustness":"7.89"},"uses_additional_data":false},{"leaderboard":"/sota/object-detection-on-lvis-v1-0-val","task":"Object Detection","dataset":"LVIS v1.0 val","model":"ViTDet-H","rank_in_archive_order":7,"of":15,"metrics":{"box AP":"53.4"},"uses_additional_data":false},{"leaderboard":"/sota/object-detection-on-lvis-v1-0-val","task":"Object Detection","dataset":"LVIS v1.0 val","model":"ViTDet-L","rank_in_archive_order":10,"of":15,"metrics":{"box AP":"51.2"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2203.16527","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.16527"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/MindCode-4/code-5/tree/main/vitdet","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kdexd/coco-rem","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/vitae-transformer/qformer","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/pwc-1/Paper-9/tree/main/1/vitdet","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/hula-ai/DAMA","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/alibaba/EasyCV","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://gitlab.com/birder/birder","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/MindCode-4/code-1/tree/main/vit","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ViTAE-Transformer/ViTDet","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/PaddlePaddle/PaddleDetection/tree/develop/configs/vitdet","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/detectron2/tree/main/projects/ViTDet","reach":null}],"summary":{"ran":1,"unverified":3},"by_repo_kind":{"listed":{"samples":4,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"893e052530d28946","entry":"process_model_info","repo":"ViTAE-Transformer/ViTDet","repo_kind":"listed","path":".dev_scripts/convert_test_benchmark_script.py","file_url":"https://github.com/ViTAE-Transformer/ViTDet/blob/HEAD/.dev_scripts/convert_test_benchmark_script.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"893e052530d28946"}},{"code_sha256_prefix":"3ccf2a44831e8069","entry":"get_num_layer_for_vit","repo":"ViTAE-Transformer/ViTDet","repo_kind":"listed","path":"mmcv_custom/layer_decay_optimizer_constructor.py","file_url":"https://github.com/ViTAE-Transformer/ViTDet/blob/HEAD/mmcv_custom/layer_decay_optimizer_constructor.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3ccf2a44831e8069"}},{"code_sha256_prefix":"8ff99a6bd2ebf598","entry":"ordered_yaml_dump","repo":"ViTAE-Transformer/ViTDet","repo_kind":"listed","path":".dev_scripts/gather_models.py","file_url":"https://github.com/ViTAE-Transformer/ViTDet/blob/HEAD/.dev_scripts/gather_models.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8ff99a6bd2ebf598"}},{"code_sha256_prefix":"138e112e22b0314e","entry":"process_checkpoint","repo":"ViTAE-Transformer/ViTDet","repo_kind":"listed","path":".dev_scripts/gather_models.py","file_url":"https://github.com/ViTAE-Transformer/ViTDet/blob/HEAD/.dev_scripts/gather_models.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"138e112e22b0314e"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}