{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/twins-revisiting-spatial-attention-design-in","title":"Twins: Revisiting the Design of Spatial Attention in Vision Transformers","arxiv_id":"2104.13840","date":"2021-04-28","proceeding":"NeurIPS 2021 12","authors":["Xiangxiang Chu","Zhi Tian","Yuqing Wang","Bo Zhang","Haibing Ren","Xiaolin Wei","Huaxia Xia","Chunhua Shen"],"abstract":"Very recently, a variety of vision transformer architectures for dense prediction tasks have been proposed and they show that the design of spatial attention is critical to their success in these tasks. In this work, we revisit the design of the spatial attention and demonstrate that a carefully-devised yet simple spatial attention mechanism performs favourably against the state-of-the-art schemes. As a result, we propose two vision transformer architectures, namely, Twins-PCPVT and Twins-SVT. Our proposed architectures are highly-efficient and easy to implement, only involving matrix multiplications that are highly optimized in modern deep learning frameworks. More importantly, the proposed architectures achieve excellent performance on a wide range of visual tasks, including image level classification as well as dense detection and segmentation. The simplicity and strong performance suggest that our proposed architectures may serve as stronger backbones for many vision tasks. Our code is released at https://github.com/Meituan-AutoML/Twins .","url_abs":"https://arxiv.org/abs/2104.13840v4","url_pdf":"https://arxiv.org/pdf/2104.13840v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"twins-revisiting-spatial-attention-design-in","repo_url":"https://github.com/Meituan-AutoML/Twins","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"twins-revisiting-spatial-attention-design-in","repo_url":"https://github.com/ahmedelmahy/myownvit","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"twins-revisiting-spatial-attention-design-in","repo_url":"https://github.com/conceptofmind/Twins-SVT-Flax","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"jax","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"twins-revisiting-spatial-attention-design-in","repo_url":"https://github.com/rwightman/pytorch-image-models","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"twins-revisiting-spatial-attention-design-in","repo_url":"https://github.com/ttt496/vit-pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"twins-revisiting-spatial-attention-design-in","repo_url":"https://github.com/xiaohu2015/pvt_detectron2","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"twins-revisiting-spatial-attention-design-in","repo_url":"https://github.com/PaddlePaddle/PaddleClas","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"paddle","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"twins-revisiting-spatial-attention-design-in","repo_url":"https://github.com/open-mmlab/mmclassification","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"twins-revisiting-spatial-attention-design-in","repo_url":"https://github.com/yangyucheng000/TwinMindSpore","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":{"status":"ok"}}],"tasks":[{"task_slug":"image-classification","task_name":"Image Classification"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"}],"methods":[{"method_slug":"attention","method_name":"Attention"},{"method_slug":"conditional-positional-encoding","method_name":"Conditional Positional Encoding"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"depthwise-convolution","method_name":"Depthwise Convolution"},{"method_slug":"global-sub-sampled-attention","method_name":"Global Sub-Sampled Attention"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"locally-grouped-self-attention","method_name":"Locally-Grouped Self-Attention"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"positional-encoding-generator","method_name":"Positional Encoding Generator"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"spatially-separable-self-attention","method_name":"Spatially Separable Self-Attention"},{"method_slug":"twins-pcpvt","method_name":"Twins-PCPVT"},{"method_slug":"twins-svt","method_name":"Twins-SVT"},{"method_slug":"vision-transformer","method_name":"Vision Transformer"}],"datasets_introduced":[],"methods_introduced":[{"slug":"global-sub-sampled-attention","name":"Global Sub-Sampled Attention","full_name":"Global Sub-Sampled Attention"},{"slug":"locally-grouped-self-attention","name":"Locally-Grouped Self-Attention","full_name":"Locally-Grouped Self-Attention"},{"slug":"spatially-separable-self-attention","name":"Spatially Separable Self-Attention","full_name":"Spatially Separable Self-Attention"},{"slug":"twins-svt","name":"Twins-SVT","full_name":"Twins-SVT"}],"results":[{"leaderboard":"/sota/image-classification-on-imagenet","task":"Image Classification","dataset":"ImageNet","model":"Twins-SVT-L","rank_in_archive_order":401,"of":1060,"metrics":{"GFLOPs":"15.1","Number of params":"99.2M","Top 1 Accuracy":"83.7%"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-ade20k","task":"Semantic Segmentation","dataset":"ADE20K","model":"Twins-SVT-L (UperNet, ImageNet-1k pretrain)","rank_in_archive_order":119,"of":235,"metrics":{"Validation mIoU":"50.2"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-ade20k-val","task":"Semantic Segmentation","dataset":"ADE20K val","model":"Twins-SVT-L (UperNet, ImageNet-1k pretrain)","rank_in_archive_order":52,"of":95,"metrics":{"mIoU":"50.2"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2104.13840","atlas_url":"https://app.syntology.ai/?focus=2104.13840","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.13840"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/rwightman/pytorch-image-models","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/xiaohu2015/pvt_detectron2","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/conceptofmind/Twins-SVT-Flax","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ttt496/vit-pytorch","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/PaddlePaddle/PaddleClas","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/open-mmlab/mmclassification","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yangyucheng000/TwinMindSpore","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ahmedelmahy/myownvit","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Meituan-AutoML/Twins","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"unverified":2},"by_repo_kind":{"official":{"samples":2,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":2,"samples":[{"code_sha256_prefix":"3bc74137ab36aa79","entry":"build_transform","repo":"Meituan-AutoML/Twins","repo_kind":"official","path":"datasets.py","file_url":"https://github.com/Meituan-AutoML/Twins/blob/HEAD/datasets.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"3bc74137ab36aa79"}},{"code_sha256_prefix":"80450600f7f09b0a","entry":"create_logger","repo":"Meituan-AutoML/Twins","repo_kind":"official","path":"logger.py","file_url":"https://github.com/Meituan-AutoML/Twins/blob/HEAD/logger.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"80450600f7f09b0a"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}