{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/dit-self-supervised-pre-training-for-document","title":"DiT: Self-supervised Pre-training for Document Image Transformer","arxiv_id":"2203.02378","date":"2022-03-04","proceeding":null,"authors":["Junlong Li","Yiheng Xu","Tengchao Lv","Lei Cui","Cha Zhang","Furu Wei"],"abstract":"Image Transformer has recently achieved significant progress for natural image understanding, either using supervised (ViT, DeiT, etc.) or self-supervised (BEiT, MAE, etc.) pre-training techniques. In this paper, we propose \\textbf{DiT}, a self-supervised pre-trained \\textbf{D}ocument \\textbf{I}mage \\textbf{T}ransformer model using large-scale unlabeled text images for Document AI tasks, which is essential since no supervised counterparts ever exist due to the lack of human-labeled document images. We leverage DiT as the backbone network in a variety of vision-based Document AI tasks, including document image classification, document layout analysis, table detection as well as text detection for OCR. Experiment results have illustrated that the self-supervised pre-trained DiT model achieves new state-of-the-art results on these downstream tasks, e.g. document image classification (91.11 $\\rightarrow$ 92.69), document layout analysis (91.0 $\\rightarrow$ 94.9), table detection (94.23 $\\rightarrow$ 96.55) and text detection for OCR (93.07 $\\rightarrow$ 94.29). The code and pre-trained models are publicly available at \\url{https://aka.ms/msdit}.","url_abs":"https://arxiv.org/abs/2203.02378v3","url_pdf":"https://arxiv.org/pdf/2203.02378v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"dit-self-supervised-pre-training-for-document","repo_url":"https://github.com/microsoft/unilm/tree/master/dit","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":null},{"paper_slug":"dit-self-supervised-pre-training-for-document","repo_url":"https://github.com/huggingface/transformers","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"dit-self-supervised-pre-training-for-document","repo_url":"https://github.com/MindCode-4/code-3/tree/main/deit","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null},{"paper_slug":"dit-self-supervised-pre-training-for-document","repo_url":"https://github.com/thibaultvt/Diard","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"document-ai","task_name":"Document AI"},{"task_slug":"document-image-classification","task_name":"Document Image Classification"},{"task_slug":"document-layout-analysis","task_name":"Document Layout Analysis"},{"task_slug":"image-classification","task_name":"Image Classification"},{"task_slug":"optical-character-recognition","task_name":"Optical Character Recognition (OCR)"},{"task_slug":"table-detection","task_name":"Table Detection"},{"task_slug":"text-detection","task_name":"Text Detection"},{"task_slug":"document-image-classification","task_name":"document-image-classification"},{"task_slug":"image-classification","task_name":"image-classification"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"deit","method_name":"DeiT"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"feedforward-network","method_name":"Feedforward Network"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"mae","method_name":"MAE"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/document-image-classification-on-rvl-cdip","task":"Document Image Classification","dataset":"RVL-CDIP","model":"DiT-L","rank_in_archive_order":22,"of":31,"metrics":{"Accuracy":"92.69%","Parameters":"304M"},"uses_additional_data":false},{"leaderboard":"/sota/document-image-classification-on-rvl-cdip","task":"Document Image Classification","dataset":"RVL-CDIP","model":"DiT-B","rank_in_archive_order":26,"of":31,"metrics":{"Accuracy":"92.11%","Parameters":"87M"},"uses_additional_data":false},{"leaderboard":"/sota/document-layout-analysis-on-publaynet-val","task":"Document Layout Analysis","dataset":"PubLayNet val","model":"DiT-L","rank_in_archive_order":7,"of":15,"metrics":{"Figure":"0.972","List":"0.960","Overall":"0.949","Table":"0.978","Text":"0.944","Title":"0.893"},"uses_additional_data":false},{"leaderboard":"/sota/table-detection-on-ctdar","task":"Table Detection","dataset":"ICDAR 2019","model":"DiT-L (Cascade)","rank_in_archive_order":1,"of":2,"metrics":{"Weighted Average F1-score":"96.55"},"uses_additional_data":false},{"leaderboard":"/sota/table-detection-on-ctdar","task":"Table Detection","dataset":"ICDAR 2019","model":"DiT-B (Cascade)","rank_in_archive_order":2,"of":2,"metrics":{"Weighted Average F1-score":"96.14"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2203.02378","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.02378"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/huggingface/transformers","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/thibaultvt/Diard","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/microsoft/unilm/tree/master/dit","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/MindCode-4/code-3/tree/main/deit","reach":null}],"summary":{"unverified":11},"by_repo_kind":{"listed":{"samples":11,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"9cf91b6b320054a8","entry":"cells_to_dataframe","repo":"thibaultvt/Diard","repo_kind":"listed","path":"modules/tables.py","file_url":"https://github.com/thibaultvt/Diard/blob/HEAD/modules/tables.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9cf91b6b320054a8"}},{"code_sha256_prefix":"97d74784c54ea294","entry":"get_cells","repo":"thibaultvt/Diard","repo_kind":"listed","path":"modules/tables.py","file_url":"https://github.com/thibaultvt/Diard/blob/HEAD/modules/tables.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"97d74784c54ea294"}},{"code_sha256_prefix":"4acc3b8aed360e21","entry":"get_html_span_by_type","repo":"thibaultvt/Diard","repo_kind":"listed","path":"modules/export.py","file_url":"https://github.com/thibaultvt/Diard/blob/HEAD/modules/export.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4acc3b8aed360e21"}},{"code_sha256_prefix":"0babed10124ceff7","entry":"get_layout_html","repo":"thibaultvt/Diard","repo_kind":"listed","path":"modules/export.py","file_url":"https://github.com/thibaultvt/Diard/blob/HEAD/modules/export.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0babed10124ceff7"}},{"code_sha256_prefix":"e59657355a3639c1","entry":"get_layout_json_dims","repo":"thibaultvt/Diard","repo_kind":"listed","path":"load_from_json.py","file_url":"https://github.com/thibaultvt/Diard/blob/HEAD/load_from_json.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e59657355a3639c1"}},{"code_sha256_prefix":"150d26752650ccb1","entry":"get_layouts_html","repo":"thibaultvt/Diard","repo_kind":"listed","path":"modules/export.py","file_url":"https://github.com/thibaultvt/Diard/blob/HEAD/modules/export.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"150d26752650ccb1"}},{"code_sha256_prefix":"4777134b6ed19f10","entry":"get_ratio","repo":"thibaultvt/Diard","repo_kind":"listed","path":"modules/sections.py","file_url":"https://github.com/thibaultvt/Diard/blob/HEAD/modules/sections.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4777134b6ed19f10"}},{"code_sha256_prefix":"07c58b9d07ef40c7","entry":"get_rows_and_columns","repo":"thibaultvt/Diard","repo_kind":"listed","path":"modules/tables.py","file_url":"https://github.com/thibaultvt/Diard/blob/HEAD/modules/tables.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"07c58b9d07ef40c7"}},{"code_sha256_prefix":"58375cbcbe5207af","entry":"map_jenks_labels","repo":"thibaultvt/Diard","repo_kind":"listed","path":"modules/sections.py","file_url":"https://github.com/thibaultvt/Diard/blob/HEAD/modules/sections.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"58375cbcbe5207af"}},{"code_sha256_prefix":"fc8daa551fdc9c9e","entry":"numbered_filenames_check","repo":"thibaultvt/Diard","repo_kind":"listed","path":"modules/document.py","file_url":"https://github.com/thibaultvt/Diard/blob/HEAD/modules/document.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fc8daa551fdc9c9e"}},{"code_sha256_prefix":"d0c31fd953920835","entry":"overlap_check","repo":"thibaultvt/Diard","repo_kind":"listed","path":"modules/document.py","file_url":"https://github.com/thibaultvt/Diard/blob/HEAD/modules/document.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d0c31fd953920835"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}