{"url":"/method/convit","slug":"convit","name":"ConViT","full_name":"ConViT","full_name_withheld":false,"description_markdown":"**ConViT** is a type of [vision transformer](https://paperswithcode.com/method/vision-transformer) that uses a gated positional self-attention module ([GPSA](https://paperswithcode.com/method/gpsa)), a form of positional self-attention which can be equipped with a “soft” convolutional inductive bias. The GPSA layers are initialized to mimic the locality of convolutional layers, then each attention head is given the freedom to escape locality by adjusting a gating parameter regulating the attention paid to position versus content information.","description_state":"present","introduced_year":null,"introduced_by":{"title":"ConViT: Improving Vision Transformers with Soft Convolutional Inductive Biases","paper":"/paper/convit-improving-vision-transformers-with","first_author":"Stéphane d'Ascoli","n_authors":6,"url_abs":null,"archive_paper_url":"https://paperswithcode.com/paper/convit-improving-vision-transformers-with"},"source":{"url":"https://arxiv.org/abs/2103.10697v2","title":"ConViT: Improving Vision Transformers with Soft Convolutional Inductive Biases","url_on_a_paper_host":true},"code_snippet_url":"https://github.com/facebookresearch/convit/blob/main/convit.py","code_snippet_url_on_a_code_host":true,"categories":[{"area":"Computer Vision","area_id":"computer-vision","collection":"Image Models","url":"/methods/category/image-models","pwc_aliases":[]},{"area":"Computer Vision","area_id":"computer-vision","collection":"Vision Transformers","url":"/methods/category/vision-transformers","pwc_aliases":["vision-transformer"]}],"n_papers_tagged":3,"archive_num_papers":3,"papers_newest_first":[{"paper":"/paper/what-do-vision-transformers-learn-a-visual","title":"What do Vision Transformers Learn? A Visual Exploration","date":"2022-12-13","arxiv_id":"2212.06727","n_code_links":1,"syntology":null},{"paper":"/paper/conviformers-convolutionally-guided-vision","title":"Conviformers: Convolutionally guided Vision Transformer","date":"2022-08-17","arxiv_id":"2208.08900","n_code_links":1,"syntology":null},{"paper":"/paper/convit-improving-vision-transformers-with","title":"ConViT: Improving Vision Transformers with Soft Convolutional Inductive Biases","date":"2021-03-19","arxiv_id":"2103.10697","n_code_links":9,"syntology":null}],"papers_shown":3,"tasks":[{"task":"/task/image-classification","name":"Image Classification","papers":2},{"task":"/task/image-classification","name":"image-classification","papers":2},{"task":"/task/fine-grained-image-classification","name":"Fine-Grained Image Classification","papers":1},{"task":"/task/inductive-bias","name":"Inductive Bias","papers":1},{"task":"/task/language-modelling","name":"Language Modelling","papers":1}],"tasks_shown":5,"n_tasks":5,"usage_by_year":[{"year":"2021","papers":1},{"year":"2022","papers":2}],"row_source":"methods_table","archive":{"source":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","archive_url":"https://paperswithcode.com/method/convit"},"syntology_read_at":"2026-09-24T18:15:14+00:00"}