{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/latr-layout-aware-transformer-for-scene-text","title":"LaTr: Layout-Aware Transformer for Scene-Text VQA","arxiv_id":"2112.12494","date":"2021-12-23","proceeding":"CVPR 2022 1","authors":["Ali Furkan Biten","Ron Litman","Yusheng Xie","Srikar Appalaraju","R. Manmatha"],"abstract":"We propose a novel multimodal architecture for Scene Text Visual Question Answering (STVQA), named Layout-Aware Transformer (LaTr). The task of STVQA requires models to reason over different modalities. Thus, we first investigate the impact of each modality, and reveal the importance of the language module, especially when enriched with layout information. Accounting for this, we propose a single objective pre-training scheme that requires only text and spatial cues. We show that applying this pre-training scheme on scanned documents has certain advantages over using natural images, despite the domain gap. Scanned documents are easy to procure, text-dense and have a variety of layouts, helping the model learn various spatial cues (e.g. left-of, below etc.) by tying together language and layout information. Compared to existing approaches, our method performs vocabulary-free decoding and, as shown, generalizes well beyond the training vocabulary. We further demonstrate that LaTr improves robustness towards OCR errors, a common reason for failure cases in STVQA. In addition, by leveraging a vision transformer, we eliminate the need for an external object detector. LaTr outperforms state-of-the-art STVQA methods on multiple datasets. In particular, +7.6% on TextVQA, +10.8% on ST-VQA and +4.0% on OCR-VQA (all absolute accuracy numbers).","url_abs":"https://arxiv.org/abs/2112.12494v2","url_pdf":"https://arxiv.org/pdf/2112.12494v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"latr-layout-aware-transformer-for-scene-text","repo_url":"https://github.com/uakarsh/latr","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"optical-character-recognition","task_name":"Optical Character Recognition (OCR)"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":null,"task_name":"TextVQA"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"},{"task_slug":"visual-question-answering","task_name":"Visual Question Answering (VQA)"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2112.12494","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2112.12494"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/uakarsh/latr","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":11},"by_repo_kind":{"listed":{"samples":11,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"0160a0950f8a408d","entry":"apply_mask_on_token_bbox","repo":"uakarsh/latr","repo_kind":"listed","path":"src/latr/utils.py","file_url":"https://github.com/uakarsh/latr/blob/HEAD/src/latr/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0160a0950f8a408d"}},{"code_sha256_prefix":"b04111d961ebf081","entry":"collate","repo":"uakarsh/latr","repo_kind":"listed","path":"src/new_latr/utils.py","file_url":"https://github.com/uakarsh/latr/blob/HEAD/src/new_latr/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b04111d961ebf081"}},{"code_sha256_prefix":"022328e95ccf040b","entry":"convert_ans_to_token","repo":"uakarsh/latr","repo_kind":"listed","path":"src/latr/utils.py","file_url":"https://github.com/uakarsh/latr/blob/HEAD/src/latr/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"022328e95ccf040b"}},{"code_sha256_prefix":"5b6c6bb7ff0f4c5f","entry":"draw_bounding_box_on_pil_image","repo":"uakarsh/latr","repo_kind":"listed","path":"src/new_latr/utils.py","file_url":"https://github.com/uakarsh/latr/blob/HEAD/src/new_latr/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5b6c6bb7ff0f4c5f"}},{"code_sha256_prefix":"819eeaf4648239c4","entry":"find_pad_idx","repo":"uakarsh/latr","repo_kind":"listed","path":"src/latr/utils.py","file_url":"https://github.com/uakarsh/latr/blob/HEAD/src/latr/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"819eeaf4648239c4"}},{"code_sha256_prefix":"8a052cb43cf0afb4","entry":"get_specific_file","repo":"uakarsh/latr","repo_kind":"listed","path":"src/latr/dataset.py","file_url":"https://github.com/uakarsh/latr/blob/HEAD/src/latr/dataset.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8a052cb43cf0afb4"}},{"code_sha256_prefix":"eeddd31a58b2af44","entry":"get_tokens_with_boxes","repo":"uakarsh/latr","repo_kind":"listed","path":"src/new_latr/dataset.py","file_url":"https://github.com/uakarsh/latr/blob/HEAD/src/new_latr/dataset.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"eeddd31a58b2af44"}},{"code_sha256_prefix":"547898699994a72b","entry":"load_json_file","repo":"uakarsh/latr","repo_kind":"listed","path":"src/latr/dataset.py","file_url":"https://github.com/uakarsh/latr/blob/HEAD/src/latr/dataset.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"547898699994a72b"}},{"code_sha256_prefix":"e596d6641d127bd8","entry":"normalize_box","repo":"uakarsh/latr","repo_kind":"listed","path":"src/new_latr/dataset.py","file_url":"https://github.com/uakarsh/latr/blob/HEAD/src/new_latr/dataset.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e596d6641d127bd8"}},{"code_sha256_prefix":"51ccac8f6cb9bd79","entry":"pad_sequence","repo":"uakarsh/latr","repo_kind":"listed","path":"src/new_latr/utils.py","file_url":"https://github.com/uakarsh/latr/blob/HEAD/src/new_latr/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"51ccac8f6cb9bd79"}},{"code_sha256_prefix":"1c118f09dd72e170","entry":"resize_align_bbox","repo":"uakarsh/latr","repo_kind":"listed","path":"src/latr/dataset.py","file_url":"https://github.com/uakarsh/latr/blob/HEAD/src/latr/dataset.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1c118f09dd72e170"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}