{"url":"/method/sparse-sinkhorn-attention","slug":"sparse-sinkhorn-attention","name":"Sparse Sinkhorn Attention","full_name":"Sparse Sinkhorn Attention","full_name_withheld":false,"description_markdown":"**Sparse Sinkhorn Attention** is an attention mechanism that reduces the memory complexity of the [dot-product attention mechanism](https://paperswithcode.com/method/scaled) and is capable of learning sparse attention outputs. It is based on the idea of differentiable sorting of internal representations within the self-attention module. SSA incorporates a meta sorting network that learns to rearrange and sort input sequences. Sinkhorn normalization is used to normalize the rows and columns of the sorting matrix. The actual SSA attention mechanism then acts on the block sorted sequences.","description_state":"present","introduced_year":null,"introduced_by":{"title":"Sparse Sinkhorn Attention","paper":"/paper/sparse-sinkhorn-attention","first_author":"Yi Tay","n_authors":5,"url_abs":null,"archive_paper_url":"https://paperswithcode.com/paper/sparse-sinkhorn-attention"},"source":{"url":"https://arxiv.org/abs/2002.11296v1","title":"Sparse Sinkhorn Attention","url_on_a_paper_host":true},"code_snippet_url":"https://github.com/lucidrains/sinkhorn-transformer/blob/3eaa76e99efeee75cf8298defaaef51621c55ff4/sinkhorn_transformer/sinkhorn_transformer.py#L286","code_snippet_url_on_a_code_host":true,"categories":[{"area":"General","area_id":"general","collection":"Attention Mechanisms","url":"/methods/category/attention-mechanisms","pwc_aliases":["attention-mechanisms-1"]}],"n_papers_tagged":2,"archive_num_papers":2,"papers_newest_first":[{"paper":"/paper/vtp-volumetric-transformer-for-multi-view","title":"VTP: Volumetric Transformer for Multi-view Multi-person 3D Pose Estimation","date":"2022-05-25","arxiv_id":"2205.12602","n_code_links":0,"syntology":null},{"paper":"/paper/sparse-sinkhorn-attention","title":"Sparse Sinkhorn Attention","date":"2020-02-26","arxiv_id":"2002.11296","n_code_links":1,"syntology":null}],"papers_shown":2,"tasks":[{"task":"/task/3d-human-pose-estimation","name":"3D Human Pose Estimation","papers":1},{"task":"/task/3d-multi-person-pose-estimation","name":"3D Multi-Person Pose Estimation","papers":1},{"task":"/task/3d-pose-estimation","name":"3D Pose Estimation","papers":1},{"task":"/task/document-classification","name":"Document Classification","papers":1},{"task":"/task/image-generation","name":"Image Generation","papers":1},{"task":"/task/language-modeling","name":"Language Modeling","papers":1},{"task":"/task/language-modelling","name":"Language Modelling","papers":1},{"task":"/task/natural-language-inference","name":"Natural Language Inference","papers":1},{"task":"/task/pose-estimation","name":"Pose Estimation","papers":1}],"tasks_shown":9,"n_tasks":9,"usage_by_year":[{"year":"2020","papers":1},{"year":"2022","papers":1}],"row_source":"methods_table","archive":{"source":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","archive_url":"https://paperswithcode.com/method/sparse-sinkhorn-attention"},"syntology_read_at":"2026-09-24T18:15:14+00:00"}