{"url":"/method/deit","slug":"deit","name":"DeiT","full_name":"Data-efficient Image Transformer","full_name_withheld":false,"description_markdown":"A **Data-Efficient Image Transformer** is a type of [Vision Transformer](https://paperswithcode.com/method/vision-transformer) for image classification tasks. The model is trained using a teacher-student strategy specific to transformers. It relies on a distillation token ensuring that the student learns from the teacher through attention.","description_state":"present","introduced_year":null,"introduced_by":{"title":null,"paper":null,"first_author":null,"n_authors":0,"url_abs":null,"archive_paper_url":null},"source":{"url":"https://arxiv.org/abs/2012.12877v2","title":"Training data-efficient image transformers & distillation through attention","url_on_a_paper_host":true},"code_snippet_url":"https://github.com/facebookresearch/deit/blob/2aefd8fc8634d099c1495ce9dba2b6c6a921d611/models.py#L16","code_snippet_url_on_a_code_host":true,"categories":[{"area":"Computer Vision","area_id":"computer-vision","collection":"Image Models","url":"/methods/category/image-models","pwc_aliases":[]}],"n_papers_tagged":93,"archive_num_papers":null,"papers_newest_first":[{"paper":"/paper/dart-differentiable-dynamic-adaptive-region","title":"DART: Differentiable Dynamic Adaptive Region Tokenizer for Vision Transformer and Mamba","date":"2025-06-12","arxiv_id":"2506.10390","n_code_links":1,"syntology":null},{"paper":null,"title":"Is Attention Required for Transformer Inference? Explore Function-preserving Attention Replacement","date":"2025-05-24","arxiv_id":"2505.21535","n_code_links":0,"syntology":null},{"paper":null,"title":"Deep learning-based identification of precipitation clouds from all-sky camera data for observatory safety","date":"2025-03-24","arxiv_id":"2503.18670","n_code_links":0,"syntology":null},{"paper":null,"title":"CAE-Net: Generalized Deepfake Image Detection using Convolution and Attention Mechanisms with Spatial and Frequency Domain Features","date":"2025-02-15","arxiv_id":"2502.10682","n_code_links":0,"syntology":null},{"paper":null,"title":"Mix-QViT: Mixed-Precision Vision Transformer Quantization Driven by Layer Importance and Quantization Sensitivity","date":"2025-01-10","arxiv_id":"2501.06357","n_code_links":0,"syntology":null},{"paper":null,"title":"Adventurer: Optimizing Vision Mamba Architecture Designs for Efficiency","date":"2025-01-01","arxiv_id":null,"n_code_links":0,"syntology":null},{"paper":"/paper/training-noise-token-pruning","title":"Training Noise Token Pruning","date":"2024-11-27","arxiv_id":"2411.18092","n_code_links":1,"syntology":null},{"paper":null,"title":"Scalable iterative pruning of large language and vision models using block coordinate descent","date":"2024-11-26","arxiv_id":"2411.17796","n_code_links":0,"syntology":null},{"paper":"/paper/pqv-mobile-a-combined-pruning-and","title":"PQV-Mobile: A Combined Pruning and Quantization Toolkit to Optimize Vision Transformers for Mobile Applications","date":"2024-08-15","arxiv_id":"2408.08437","n_code_links":1,"syntology":null},{"paper":"/paper/mixed-non-linear-quantization-for-vision","title":"Mixed Non-linear Quantization for Vision Transformers","date":"2024-07-26","arxiv_id":"2407.18437","n_code_links":1,"syntology":null},{"paper":"/paper/knowledge-distillation-to-effectively-attain","title":"Knowledge distillation to effectively attain both region-of-interest and global semantics from an image where multiple objects appear","date":"2024-07-11","arxiv_id":"2407.08257","n_code_links":1,"syntology":null},{"paper":"/paper/leveraging-transformers-for-weakly-supervised","title":"Leveraging Transformers for Weakly Supervised Object Localization in Unconstrained Videos","date":"2024-07-08","arxiv_id":"2407.06018","n_code_links":1,"syntology":null},{"paper":null,"title":"Predicting Depression and Anxiety Risk in Dutch Neighborhoods from Street-View Images","date":"2024-06-27","arxiv_id":"2407.09547","n_code_links":0,"syntology":null},{"paper":"/paper/liere-generalizing-rotary-position-encodings","title":"LieRE: Generalizing Rotary Position Encodings","date":"2024-06-14","arxiv_id":"2406.10322","n_code_links":1,"syntology":{"ran":1,"of":3,"unverified":2,"pointer_only":0}},{"paper":"/paper/decomposing-and-interpreting-image","title":"Decomposing and Interpreting Image Representations via Text in ViTs Beyond CLIP","date":"2024-06-03","arxiv_id":"2406.01583","n_code_links":1,"syntology":{"ran":10,"of":13,"unverified":3,"pointer_only":13}},{"paper":"/paper/a-general-and-efficient-training-for","title":"A General and Efficient Training for Transformer via Token Expansion","date":"2024-03-31","arxiv_id":"2404.00672","n_code_links":1,"syntology":{"ran":7,"of":8,"unverified":1,"pointer_only":0}},{"paper":"/paper/multi-criteria-token-fusion-with-one-step","title":"Multi-criteria Token Fusion with One-step-ahead Attention for Efficient Vision Transformers","date":"2024-03-15","arxiv_id":"2403.10030","n_code_links":1,"syntology":{"ran":3,"of":5,"unverified":2,"pointer_only":0}},{"paper":null,"title":"MPTQ-ViT: Mixed-Precision Post-Training Quantization for Vision Transformer","date":"2024-01-26","arxiv_id":"2401.14895","n_code_links":0,"syntology":null},{"paper":null,"title":"Detecting and recognizing characters in Greek papyri with YOLOv8, DeiT and SimCLR","date":"2024-01-23","arxiv_id":"2401.12513","n_code_links":0,"syntology":null},{"paper":null,"title":"LRP-QViT: Mixed-Precision Vision Transformer Quantization via Layer-wise Relevance Propagation","date":"2024-01-20","arxiv_id":"2401.11243","n_code_links":0,"syntology":null},{"paper":"/paper/vision-mamba-efficient-visual-representation","title":"Vision Mamba: Efficient Visual Representation Learning with Bidirectional State Space Model","date":"2024-01-17","arxiv_id":"2401.09417","n_code_links":15,"syntology":{"ran":2,"of":13,"unverified":11,"pointer_only":1}},{"paper":null,"title":"TPC-ViT: Token Propagation Controller for Efficient Vision Transformer","date":"2024-01-03","arxiv_id":"2401.01470","n_code_links":0,"syntology":null},{"paper":null,"title":"ROI-Aware Multiscale Cross-Attention Vision Transformer for Pest Image Identification","date":"2023-12-28","arxiv_id":"2312.16914","n_code_links":0,"syntology":null},{"paper":null,"title":"Bridging The Gaps Between Token Pruning and Full Pre-training via Masked Fine-tuning","date":"2023-10-26","arxiv_id":"2310.17177","n_code_links":0,"syntology":null},{"paper":null,"title":"USDC: Unified Static and Dynamic Compression for Visual Transformer","date":"2023-10-17","arxiv_id":"2310.11117","n_code_links":0,"syntology":null},{"paper":null,"title":"Accelerating Vision Transformers Based on Heterogeneous Attention Patterns","date":"2023-10-11","arxiv_id":"2310.07664","n_code_links":0,"syntology":null},{"paper":null,"title":"EWasteNet: A Two-Stream Data Efficient Image Transformer Approach for E-Waste Classification","date":"2023-09-28","arxiv_id":"2311.12823","n_code_links":0,"syntology":null},{"paper":null,"title":"Multi-Dimensional Hyena for Spatial Inductive Bias","date":"2023-09-24","arxiv_id":"2309.13600","n_code_links":0,"syntology":null},{"paper":"/paper/weight-averaging-improves-knowledge","title":"Weight Averaging Improves Knowledge Distillation under Domain Shift","date":"2023-09-20","arxiv_id":"2309.11446","n_code_links":1,"syntology":null},{"paper":"/paper/tuning-pre-trained-model-via-moment-probing","title":"Tuning Pre-trained Model via Moment Probing","date":"2023-07-21","arxiv_id":"2307.11342","n_code_links":1,"syntology":{"ran":8,"of":13,"unverified":5,"pointer_only":13}}],"papers_shown":30,"tasks":[{"task":"/task/image-classification","name":"Image Classification","papers":25},{"task":"/task/image-classification","name":"image-classification","papers":16},{"task":"/task/object-detection","name":"Object Detection","papers":11},{"task":"/task/quantization","name":"Quantization","papers":10},{"task":"/task/object-detection-1","name":"object-detection","papers":10},{"task":"/task/semantic-segmentation","name":"Semantic Segmentation","papers":8},{"task":"/task/inductive-bias","name":"Inductive Bias","papers":6},{"task":null,"name":"Position","papers":5},{"task":"/task/self-supervised-learning","name":"Self-Supervised Learning","papers":5},{"task":"/task/efficient-vits","name":"Efficient ViTs","papers":4},{"task":"/task/fine-grained-image-classification","name":"Fine-Grained Image Classification","papers":4},{"task":"/task/knowledge-distillation","name":"Knowledge Distillation","papers":4},{"task":"/task/data-augmentation","name":"Data Augmentation","papers":3},{"task":"/task/document-image-classification","name":"Document Image Classification","papers":3},{"task":"/task/document-layout-analysis","name":"Document Layout Analysis","papers":3},{"task":"/task/mamba","name":"Mamba","papers":3},{"task":"/task/architecture-search","name":"Neural Architecture Search","papers":3},{"task":"/task/representation-learning","name":"Representation Learning","papers":3},{"task":"/task/all","name":"All","papers":2},{"task":"/task/automl","name":"AutoML","papers":2}],"tasks_shown":20,"n_tasks":78,"usage_by_year":[{"year":"2020","papers":1},{"year":"2021","papers":27},{"year":"2022","papers":23},{"year":"2023","papers":20},{"year":"2024","papers":16},{"year":"2025","papers":6}],"row_source":"embedded","archive":{"source":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","archive_url":"https://paperswithcode.com/method/deit"},"syntology_read_at":"2026-09-24T18:15:14+00:00"}