{"url":"/method/single-headed-attention","slug":"single-headed-attention","name":"Single-Headed Attention","full_name":"Single-Headed Attention","full_name_withheld":false,"description_markdown":"**Single-Headed Attention** is a single-headed attention module used in the [SHA-RNN](https://paperswithcode.com/method/sha-rnn) language model. The principle design reasons for single-headedness were simplicity (avoiding running out of memory) and scepticism about the benefits of using multiple heads.","description_state":"present","introduced_year":null,"introduced_by":{"title":"Single Headed Attention RNN: Stop Thinking With Your Head","paper":"/paper/single-headed-attention-rnn-stop-thinking","first_author":"Stephen Merity","n_authors":1,"url_abs":null,"archive_paper_url":"https://paperswithcode.com/paper/single-headed-attention-rnn-stop-thinking"},"source":{"url":"https://arxiv.org/abs/1911.11423v2","title":"Single Headed Attention RNN: Stop Thinking With Your Head","url_on_a_paper_host":true},"code_snippet_url":"https://github.com/Smerity/sha-rnn/blob/218d748022dbcf32d50bbbb4d151a9b6de3f8bba/model.py#L53","code_snippet_url_on_a_code_host":true,"categories":[{"area":"General","area_id":"general","collection":"Attention Modules","url":"/methods/category/attention-modules","pwc_aliases":[]}],"n_papers_tagged":6,"archive_num_papers":6,"papers_newest_first":[{"paper":null,"title":"Meta-Learning-Based Delayless Subband Adaptive Filter using Complex Self-Attention for Active Noise Control","date":"2024-12-27","arxiv_id":"2412.19471","n_code_links":0,"syntology":null},{"paper":null,"title":"SHAQ: Single Headed Attention with Quasi-Recurrence","date":"2021-08-18","arxiv_id":"2108.08207","n_code_links":0,"syntology":null},{"paper":"/paper/deep-diacritization-efficient-hierarchical","title":"Deep Diacritization: Efficient Hierarchical Recurrence for Improved Arabic Diacritization","date":"2020-11-01","arxiv_id":"2011.00538","n_code_links":1,"syntology":null},{"paper":"/paper/attention-based-joint-detection-of-object-and","title":"Attention-based Joint Detection of Object and Semantic Part","date":"2020-07-05","arxiv_id":"2007.02419","n_code_links":1,"syntology":{"ran":3,"of":12,"unverified":9,"pointer_only":0}},{"paper":"/paper/single-headed-attention-rnn-stop-thinking","title":"Single Headed Attention RNN: Stop Thinking With Your Head","date":"2019-11-26","arxiv_id":"1911.11423","n_code_links":5,"syntology":null},{"paper":"/paper/attention-and-lexicon-regularized-lstm-for","title":"Attention and Lexicon Regularized LSTM for Aspect-based Sentiment Analysis","date":"2019-07-01","arxiv_id":null,"n_code_links":1,"syntology":null}],"papers_shown":6,"tasks":[{"task":"/task/sentence","name":"Sentence","papers":2},{"task":"/task/arabic-text-diacritization","name":"Arabic Text Diacritization","papers":1},{"task":"/task/aspect-based-sentiment-analysis-1","name":"Aspect-Based Sentiment Analysis","papers":1},{"task":"/task/aspect-based-sentiment-analysis","name":"Aspect-Based Sentiment Analysis (ABSA)","papers":1},{"task":"/task/decoder","name":"Decoder","papers":1},{"task":null,"name":"GPU","papers":1},{"task":"/task/hyperparameter-optimization","name":"Hyperparameter Optimization","papers":1},{"task":"/task/language-modeling","name":"Language Modeling","papers":1},{"task":"/task/language-modelling","name":"Language Modelling","papers":1},{"task":"/task/meta-learning","name":"Meta-Learning","papers":1},{"task":"/task/object","name":"Object","papers":1},{"task":"/task/object-detection","name":"Object Detection","papers":1},{"task":"/task/semantic-part-detection","name":"Semantic Part Detection","papers":1},{"task":"/task/sentiment-analysis","name":"Sentiment Analysis","papers":1},{"task":"/task/object-detection-1","name":"object-detection","papers":1},{"task":"/task/regression-1","name":"regression","papers":1},{"task":null,"name":"valid","papers":1}],"tasks_shown":17,"n_tasks":17,"usage_by_year":[{"year":"2019","papers":2},{"year":"2020","papers":2},{"year":"2021","papers":1},{"year":"2024","papers":1}],"row_source":"methods_table","archive":{"source":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","archive_url":"https://paperswithcode.com/method/single-headed-attention"},"syntology_read_at":"2026-09-24T18:15:14+00:00"}