{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/shvit-single-head-vision-transformer-with","title":"SHViT: Single-Head Vision Transformer with Memory Efficient Macro Design","arxiv_id":"2401.16456","date":"2024-01-29","proceeding":"CVPR 2024 1","authors":["Seokju Yun","Youngmin Ro"],"abstract":"Recently, efficient Vision Transformers have shown great performance with low latency on resource-constrained devices. Conventionally, they use 4x4 patch embeddings and a 4-stage structure at the macro level, while utilizing sophisticated attention with multi-head configuration at the micro level. This paper aims to address computational redundancy at all design levels in a memory-efficient manner. We discover that using larger-stride patchify stem not only reduces memory access costs but also achieves competitive performance by leveraging token representations with reduced spatial redundancy from the early stages. Furthermore, our preliminary analyses suggest that attention layers in the early stages can be substituted with convolutions, and several attention heads in the latter stages are computationally redundant. To handle this, we introduce a single-head attention module that inherently prevents head redundancy and simultaneously boosts accuracy by parallelly combining global and local information. Building upon our solutions, we introduce SHViT, a Single-Head Vision Transformer that obtains the state-of-the-art speed-accuracy tradeoff. For example, on ImageNet-1k, our SHViT-S4 is 3.3x, 8.1x, and 2.4x faster than MobileViTv2 x1.0 on GPU, CPU, and iPhone12 mobile device, respectively, while being 1.3% more accurate. For object detection and instance segmentation on MS COCO using Mask-RCNN head, our model achieves performance comparable to FastViT-SA12 while exhibiting 3.8x and 2.0x lower backbone latency on GPU and mobile device, respectively.","url_abs":"https://arxiv.org/abs/2401.16456v2","url_pdf":"https://arxiv.org/pdf/2401.16456v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"shvit-single-head-vision-transformer-with","repo_url":"https://github.com/ysj9909/SHViT","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":null,"task_name":"CPU"},{"task_slug":null,"task_name":"GPU"},{"task_slug":"image-classification","task_name":"Image Classification"},{"task_slug":"instance-segmentation","task_name":"Instance Segmentation"},{"task_slug":"object-detection","task_name":"Object Detection"},{"task_slug":"object-detection-1","task_name":"object-detection"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"mobilevitv2","method_name":"MobileViTv2"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"},{"method_slug":"vision-transformer","method_name":"Vision Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2401.16456","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.16456"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ysj9909/SHViT","reach":null}],"summary":{"ran":7,"unverified":1},"by_repo_kind":{"official":{"samples":8,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":8,"samples":[{"code_sha256_prefix":"66027e67376f823d","entry":"BasicBlock","repo":"ysj9909/SHViT","repo_kind":"official","path":"model/shvit.py","file_url":"https://github.com/ysj9909/SHViT/blob/HEAD/model/shvit.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"66027e67376f823d"}},{"code_sha256_prefix":"c8d049438a0879cb","entry":"Conv2d_BN","repo":"ysj9909/SHViT","repo_kind":"official","path":"model/shvit.py","file_url":"https://github.com/ysj9909/SHViT/blob/HEAD/model/shvit.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"c8d049438a0879cb"}},{"code_sha256_prefix":"f3619d88bbb551e1","entry":"FFN","repo":"ysj9909/SHViT","repo_kind":"official","path":"model/shvit.py","file_url":"https://github.com/ysj9909/SHViT/blob/HEAD/model/shvit.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"f3619d88bbb551e1"}},{"code_sha256_prefix":"40883b88da57364e","entry":"GroupNorm","repo":"ysj9909/SHViT","repo_kind":"official","path":"model/shvit.py","file_url":"https://github.com/ysj9909/SHViT/blob/HEAD/model/shvit.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"40883b88da57364e"}},{"code_sha256_prefix":"7f0dd3b9287448aa","entry":"PatchMerging","repo":"ysj9909/SHViT","repo_kind":"official","path":"model/shvit.py","file_url":"https://github.com/ysj9909/SHViT/blob/HEAD/model/shvit.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"7f0dd3b9287448aa"}},{"code_sha256_prefix":"61f485bf32a93933","entry":"Residual","repo":"ysj9909/SHViT","repo_kind":"official","path":"model/shvit.py","file_url":"https://github.com/ysj9909/SHViT/blob/HEAD/model/shvit.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"61f485bf32a93933"}},{"code_sha256_prefix":"2de99ffa1cb70568","entry":"SHSA","repo":"ysj9909/SHViT","repo_kind":"official","path":"model/shvit.py","file_url":"https://github.com/ysj9909/SHViT/blob/HEAD/model/shvit.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"2de99ffa1cb70568"}},{"code_sha256_prefix":"6d33b8f4ae0ebedb","entry":"SHViT","repo":"ysj9909/SHViT","repo_kind":"official","path":"model/shvit.py","file_url":"https://github.com/ysj9909/SHViT/blob/HEAD/model/shvit.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"6d33b8f4ae0ebedb"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}