{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/unifying-vision-text-and-layout-for-universal","title":"Unifying Vision, Text, and Layout for Universal Document Processing","arxiv_id":"2212.02623","date":"2022-12-05","proceeding":"CVPR 2023 1","authors":["Zineng Tang","ZiYi Yang","Guoxin Wang","Yuwei Fang","Yang Liu","Chenguang Zhu","Michael Zeng","Cha Zhang","Mohit Bansal"],"abstract":"We propose Universal Document Processing (UDOP), a foundation Document AI model which unifies text, image, and layout modalities together with varied task formats, including document understanding and generation. UDOP leverages the spatial correlation between textual content and document image to model image, text, and layout modalities with one uniform representation. With a novel Vision-Text-Layout Transformer, UDOP unifies pretraining and multi-domain downstream tasks into a prompt-based sequence generation scheme. UDOP is pretrained on both large-scale unlabeled document corpora using innovative self-supervised objectives and diverse labeled data. UDOP also learns to generate document images from text and layout modalities via masked image reconstruction. To the best of our knowledge, this is the first time in the field of document AI that one model simultaneously achieves high-quality neural document editing and content customization. Our method sets the state-of-the-art on 8 Document AI tasks, e.g., document understanding and QA, across diverse data domains like finance reports, academic papers, and websites. UDOP ranks first on the leaderboard of the Document Understanding Benchmark.","url_abs":"https://arxiv.org/abs/2212.02623v3","url_pdf":"https://arxiv.org/pdf/2212.02623v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"unifying-vision-text-and-layout-for-universal","repo_url":"https://github.com/microsoft/i-code","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"jax","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"unifying-vision-text-and-layout-for-universal","repo_url":"https://github.com/microsoft/udop","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"unifying-vision-text-and-layout-for-universal","repo_url":"https://github.com/DS4SD/MarkushGrapher","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"unifying-vision-text-and-layout-for-universal","repo_url":"https://github.com/MindCode-4/code-5/tree/main/udop","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null},{"paper_slug":"unifying-vision-text-and-layout-for-universal","repo_url":"https://github.com/pwc-1/Paper-10","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"document-ai","task_name":"Document AI"},{"task_slug":"image-reconstruction","task_name":"Image Reconstruction"},{"task_slug":"visual-question-answering","task_name":"Visual Question Answering (VQA)"},{"task_slug":"document-understanding","task_name":"document understanding"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/visual-question-answering-on-docvqa-test","task":"Visual Question Answering (VQA)","dataset":"DocVQA test","model":"UDOP (aux)","rank_in_archive_order":11,"of":33,"metrics":{"ANLS":"0.878"},"uses_additional_data":true},{"leaderboard":"/sota/visual-question-answering-on-docvqa-test","task":"Visual Question Answering (VQA)","dataset":"DocVQA test","model":"UDOP","rank_in_archive_order":17,"of":33,"metrics":{"ANLS":"0.847"},"uses_additional_data":false},{"leaderboard":"/sota/visual-question-answering-vqa-on","task":"Visual Question Answering (VQA)","dataset":"InfographicVQA","model":"UDOP (aux)","rank_in_archive_order":5,"of":21,"metrics":{"ANLS":"63.0"},"uses_additional_data":true},{"leaderboard":"/sota/visual-question-answering-vqa-on","task":"Visual Question Answering (VQA)","dataset":"InfographicVQA","model":"UDOP","rank_in_archive_order":16,"of":21,"metrics":{"ANLS":"47.4"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2212.02623","atlas_url":"https://app.syntology.ai/?focus=2212.02623","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2212.02623"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/MindCode-4/code-5/tree/main/udop","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/pwc-1/Paper-10","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/DS4SD/MarkushGrapher","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/microsoft/i-code","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/microsoft/udop","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":11,"ran_violates":2,"ran_draft_wrong":1,"unverified":3},"by_repo_kind":{"listed":{"samples":14,"ran":11,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":3,"samples":[{"code_sha256_prefix":"f587be8f27266175","entry":"custom_huber","repo":"DS4SD/MarkushGrapher","repo_kind":"listed","path":"markushgrapher/core/trainers/losses.py","file_url":"https://github.com/DS4SD/MarkushGrapher/blob/HEAD/markushgrapher/core/trainers/losses.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f587be8f27266175"}},{"code_sha256_prefix":"8ad08bf63e399fe5","entry":"custom_huber2","repo":"DS4SD/MarkushGrapher","repo_kind":"listed","path":"markushgrapher/core/trainers/losses.py","file_url":"https://github.com/DS4SD/MarkushGrapher/blob/HEAD/markushgrapher/core/trainers/losses.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8ad08bf63e399fe5"}},{"code_sha256_prefix":"424012cb37b31172","entry":"default","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"424012cb37b31172"}},{"code_sha256_prefix":"68402efebc760365","entry":"estimate_word_width","repo":"DS4SD/MarkushGrapher","repo_kind":"listed","path":"markushgrapher/core/common/data_preprocessing.py","file_url":"https://github.com/DS4SD/MarkushGrapher/blob/HEAD/markushgrapher/core/common/data_preprocessing.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"68402efebc760365"}},{"code_sha256_prefix":"aa5486a3650902d8","entry":"exists","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"aa5486a3650902d8"}},{"code_sha256_prefix":"9f1130d32c6430f8","entry":"get_last_checkpoint","repo":"DS4SD/MarkushGrapher","repo_kind":"listed","path":"markushgrapher/core/common/utils.py","file_url":"https://github.com/DS4SD/MarkushGrapher/blob/HEAD/markushgrapher/core/common/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9f1130d32c6430f8"}},{"code_sha256_prefix":"a3b662bbc3a23327","entry":"get_linear_with_fact_schedule_with_warmup","repo":"DS4SD/MarkushGrapher","repo_kind":"listed","path":"markushgrapher/core/trainers/optimization.py","file_url":"https://github.com/DS4SD/MarkushGrapher/blob/HEAD/markushgrapher/core/trainers/optimization.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a3b662bbc3a23327"}},{"code_sha256_prefix":"e78d0e6d79459413","entry":"get_scheduler","repo":"DS4SD/MarkushGrapher","repo_kind":"listed","path":"markushgrapher/core/trainers/optimization.py","file_url":"https://github.com/DS4SD/MarkushGrapher/blob/HEAD/markushgrapher/core/trainers/optimization.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e78d0e6d79459413"}},{"code_sha256_prefix":"a83a110a9551382d","entry":"huber_loss","repo":"DS4SD/MarkushGrapher","repo_kind":"listed","path":"markushgrapher/core/trainers/losses.py","file_url":"https://github.com/DS4SD/MarkushGrapher/blob/HEAD/markushgrapher/core/trainers/losses.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a83a110a9551382d"}},{"code_sha256_prefix":"4d78a796062fd89b","entry":"pad_sequence_native","repo":"DS4SD/MarkushGrapher","repo_kind":"listed","path":"markushgrapher/core/trainers/data_collator.py","file_url":"https://github.com/DS4SD/MarkushGrapher/blob/HEAD/markushgrapher/core/trainers/data_collator.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4d78a796062fd89b"}},{"code_sha256_prefix":"fbd45e87bae891a7","entry":"setup_logging","repo":"DS4SD/MarkushGrapher","repo_kind":"listed","path":"markushgrapher/core/common/begin.py","file_url":"https://github.com/DS4SD/MarkushGrapher/blob/HEAD/markushgrapher/core/common/begin.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fbd45e87bae891a7"}},{"code_sha256_prefix":"084ee3628fc5d124","entry":"split_bounding_box_for_words","repo":"DS4SD/MarkushGrapher","repo_kind":"listed","path":"markushgrapher/core/common/data_preprocessing.py","file_url":"https://github.com/DS4SD/MarkushGrapher/blob/HEAD/markushgrapher/core/common/data_preprocessing.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"084ee3628fc5d124"}},{"code_sha256_prefix":"32b4122ad5350b54","entry":"split_sentence_into_words","repo":"DS4SD/MarkushGrapher","repo_kind":"listed","path":"markushgrapher/core/common/data_preprocessing.py","file_url":"https://github.com/DS4SD/MarkushGrapher/blob/HEAD/markushgrapher/core/common/data_preprocessing.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"32b4122ad5350b54"}},{"code_sha256_prefix":"9a299fe5ae09e407","entry":"uniq","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"9a299fe5ae09e407"}},{"code_sha256_prefix":"ea1bb4ac7f48a7b5","entry":"calculate_iou","repo":"DS4SD/MarkushGrapher","repo_kind":"listed","path":"markushgrapher/core/common/utils.py","file_url":"https://github.com/DS4SD/MarkushGrapher/blob/HEAD/markushgrapher/core/common/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ea1bb4ac7f48a7b5"}},{"code_sha256_prefix":"59eacee354fa7772","entry":"clamp","repo":"DS4SD/MarkushGrapher","repo_kind":"listed","path":"markushgrapher/core/common/utils.py","file_url":"https://github.com/DS4SD/MarkushGrapher/blob/HEAD/markushgrapher/core/common/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"59eacee354fa7772"}},{"code_sha256_prefix":"5b942743f6a5251f","entry":"last_checkpoint","repo":"DS4SD/MarkushGrapher","repo_kind":"listed","path":"markushgrapher/core/common/begin.py","file_url":"https://github.com/DS4SD/MarkushGrapher/blob/HEAD/markushgrapher/core/common/begin.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5b942743f6a5251f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}