{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/sima-simple-softmax-free-attention-for-vision","title":"SimA: Simple Softmax-free Attention for Vision Transformers","arxiv_id":"2206.08898","date":"2022-06-17","proceeding":null,"authors":["Soroush Abbasi Koohpayegani","Hamed Pirsiavash"],"abstract":"Recently, vision transformers have become very popular. However, deploying them in many applications is computationally expensive partly due to the Softmax layer in the attention block. We introduce a simple but effective, Softmax-free attention block, SimA, which normalizes query and key matrices with simple $\\ell_1$-norm instead of using Softmax layer. Then, the attention block in SimA is a simple multiplication of three matrices, so SimA can dynamically change the ordering of the computation at the test time to achieve linear computation on the number of tokens or the number of channels. We empirically show that SimA applied to three SOTA variations of transformers, DeiT, XCiT, and CvT, results in on-par accuracy compared to the SOTA models, without any need for Softmax layer. Interestingly, changing SimA from multi-head to single-head has only a small effect on the accuracy, which simplifies the attention block further. The code is available here: https://github.com/UCDvision/sima","url_abs":"https://arxiv.org/abs/2206.08898v2","url_pdf":"https://arxiv.org/pdf/2206.08898v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"sima-simple-softmax-free-attention-for-vision","repo_url":"https://github.com/ucdvision/sima","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[],"methods":[{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"average-pooling","method_name":"Average Pooling"},{"method_slug":"batch-normalization","method_name":"Batch Normalization"},{"method_slug":"convolution","method_name":"Convolution"},{"method_slug":"cross-covariance-attention","method_name":"Cross-Covariance Attention"},{"method_slug":"cvt","method_name":"CvT"},{"method_slug":"deit","method_name":"DeiT"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"depthwise-convolution","method_name":"Depthwise Convolution"},{"method_slug":"depthwise-separable-convolution","method_name":"Depthwise Separable Convolution"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"feedforward-network","method_name":"Feedforward Network"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"local-patch-interaction","method_name":"Local Patch Interaction"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"pointwise-convolution","method_name":"Pointwise Convolution"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"test","method_name":"Test"},{"method_slug":"xcit","method_name":"XCiT"},{"method_slug":"xcit-layer","method_name":"XCiT Layer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2206.08898","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2206.08898"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ucdvision/sima","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":3},"by_repo_kind":{"official":{"samples":3,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"18119fb5f5299851","entry":"build_dataset","repo":"ucdvision/sima","repo_kind":"official","path":"datasets.py","file_url":"https://github.com/ucdvision/sima/blob/HEAD/datasets.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"18119fb5f5299851"}},{"code_sha256_prefix":"e7791ea55637b67a","entry":"build_transform","repo":"ucdvision/sima","repo_kind":"official","path":"datasets.py","file_url":"https://github.com/ucdvision/sima/blob/HEAD/datasets.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e7791ea55637b67a"}},{"code_sha256_prefix":"429abe9dc8793925","entry":"conv3x3","repo":"ucdvision/sima","repo_kind":"official","path":"sima.py","file_url":"https://github.com/ucdvision/sima/blob/HEAD/sima.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"429abe9dc8793925"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}