{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/structurallm-structural-pre-training-for-form","title":"StructuralLM: Structural Pre-training for Form Understanding","arxiv_id":"2105.11210","date":"2021-05-24","proceeding":"ACL 2021 5","authors":["Chenliang Li","Bin Bi","Ming Yan","Wei Wang","Songfang Huang","Fei Huang","Luo Si"],"abstract":"Large pre-trained language models achieve state-of-the-art results when fine-tuned on downstream NLP tasks. However, they almost exclusively focus on text-only representation, while neglecting cell-level layout information that is important for form image understanding. In this paper, we propose a new pre-training approach, StructuralLM, to jointly leverage cell and layout information from scanned documents. Specifically, we pre-train StructuralLM with two new designs to make the most of the interactions of cell and layout information: 1) each cell as a semantic unit; 2) classification of cell positions. The pre-trained StructuralLM achieves new state-of-the-art results in different types of downstream tasks, including form understanding (from 78.95 to 85.14), document visual question answering (from 72.59 to 83.94) and document image classification (from 94.43 to 96.08).","url_abs":"https://arxiv.org/abs/2105.11210v1","url_pdf":"https://arxiv.org/pdf/2105.11210v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"structurallm-structural-pre-training-for-form","repo_url":"https://github.com/alibaba/AliceMind","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"document-image-classification","task_name":"Document Image Classification"},{"task_slug":"form","task_name":"Form"},{"task_slug":"image-classification","task_name":"Image Classification"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"},{"task_slug":"visual-question-answering","task_name":"Visual Question Answering (VQA)"},{"task_slug":"document-image-classification","task_name":"document-image-classification"},{"task_slug":"image-classification","task_name":"image-classification"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2105.11210","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2105.11210"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/alibaba/AliceMind","reach":null}],"summary":{"unverified":5},"by_repo_kind":{"official":{"samples":5,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"0a494585f68860d3","entry":"create_initializer","repo":"alibaba/AliceMind","repo_kind":"official","path":"StructuralLM/modeling.py","file_url":"https://github.com/alibaba/AliceMind/blob/HEAD/StructuralLM/modeling.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0a494585f68860d3"}},{"code_sha256_prefix":"ea30bd05c67b9171","entry":"embedding_postprocessor","repo":"alibaba/AliceMind","repo_kind":"official","path":"StructuralLM/modeling.py","file_url":"https://github.com/alibaba/AliceMind/blob/HEAD/StructuralLM/modeling.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ea30bd05c67b9171"}},{"code_sha256_prefix":"180084952859e5e0","entry":"layer_norm","repo":"alibaba/AliceMind","repo_kind":"official","path":"StructuralLM/modeling.py","file_url":"https://github.com/alibaba/AliceMind/blob/HEAD/StructuralLM/modeling.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"180084952859e5e0"}},{"code_sha256_prefix":"58bed10d69223f15","entry":"layer_norm_and_dropout","repo":"alibaba/AliceMind","repo_kind":"official","path":"StructuralLM/modeling.py","file_url":"https://github.com/alibaba/AliceMind/blob/HEAD/StructuralLM/modeling.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"58bed10d69223f15"}},{"code_sha256_prefix":"e64a2b41596499ba","entry":"position_embedding_lookup","repo":"alibaba/AliceMind","repo_kind":"official","path":"StructuralLM/modeling.py","file_url":"https://github.com/alibaba/AliceMind/blob/HEAD/StructuralLM/modeling.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e64a2b41596499ba"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}