{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/structural-guidance-for-transformer-language","title":"Structural Guidance for Transformer Language Models","arxiv_id":"2108.00104","date":"2021-07-30","proceeding":"ACL 2021 5","authors":["Peng Qian","Tahira Naseem","Roger Levy","Ramón Fernandez Astudillo"],"abstract":"Transformer-based language models pre-trained on large amounts of text data have proven remarkably successful in learning generic transferable linguistic representations. Here we study whether structural guidance leads to more human-like systematic linguistic generalization in Transformer language models without resorting to pre-training on very large amounts of data. We explore two general ideas. The \"Generative Parsing\" idea jointly models the incremental parse and word sequence as part of the same sequence modeling task. The \"Structural Scaffold\" idea guides the language model's representation via additional structure loss that separately predicts the incremental constituency parse. We train the proposed models along with a vanilla Transformer language model baseline on a 14 million-token and a 46 million-token subset of the BLLIP dataset, and evaluate models' syntactic generalization performances on SG Test Suites and sized BLiMP. Experiment results across two benchmarks suggest converging evidence that generative structural supervisions can induce more robust and humanlike linguistic generalization in Transformer language models without the need for data intensive pre-training.","url_abs":"https://arxiv.org/abs/2108.00104v1","url_pdf":"https://arxiv.org/pdf/2108.00104v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"structural-guidance-for-transformer-language","repo_url":"https://github.com/IBM/transformers-struct-guidance","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2108.00104","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2108.00104"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/IBM/transformers-struct-guidance","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":1,"ran_violates":1,"unverified":10},"by_repo_kind":{"official":{"samples":12,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"2347da1339a984a0","entry":"get_between_brackets","repo":"IBM/transformers-struct-guidance","repo_kind":"official","path":"src/get_oracle.py","file_url":"https://github.com/IBM/transformers-struct-guidance/blob/HEAD/src/get_oracle.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2347da1339a984a0"}},{"code_sha256_prefix":"625f98c10e41a690","entry":"is_next_open_bracket","repo":"IBM/transformers-struct-guidance","repo_kind":"official","path":"src/get_oracle.py","file_url":"https://github.com/IBM/transformers-struct-guidance/blob/HEAD/src/get_oracle.py","link_basis":"plan_row","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"625f98c10e41a690"}},{"code_sha256_prefix":"ec290339cc73f6e4","entry":"get_batches","repo":"IBM/transformers-struct-guidance","repo_kind":"official","path":"src/lm.py","file_url":"https://github.com/IBM/transformers-struct-guidance/blob/HEAD/src/lm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ec290339cc73f6e4"}},{"code_sha256_prefix":"efaab57473a43333","entry":"is_nonterminal","repo":"IBM/transformers-struct-guidance","repo_kind":"official","path":"src/lm-sc.py","file_url":"https://github.com/IBM/transformers-struct-guidance/blob/HEAD/src/lm-sc.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"efaab57473a43333"}},{"code_sha256_prefix":"46da2d8a70506f38","entry":"is_nt","repo":"IBM/transformers-struct-guidance","repo_kind":"official","path":"src/get_action_ngram_list.py","file_url":"https://github.com/IBM/transformers-struct-guidance/blob/HEAD/src/get_action_ngram_list.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"46da2d8a70506f38"}},{"code_sha256_prefix":"0adfa8c374ac882f","entry":"is_reduce","repo":"IBM/transformers-struct-guidance","repo_kind":"official","path":"src/get_action_ngram_list.py","file_url":"https://github.com/IBM/transformers-struct-guidance/blob/HEAD/src/get_action_ngram_list.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0adfa8c374ac882f"}},{"code_sha256_prefix":"7f5c8e83b0117a9f","entry":"is_valid_action_sequence","repo":"IBM/transformers-struct-guidance","repo_kind":"official","path":"src/get_oracle.py","file_url":"https://github.com/IBM/transformers-struct-guidance/blob/HEAD/src/get_oracle.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7f5c8e83b0117a9f"}},{"code_sha256_prefix":"b0e89cf46d9f15be","entry":"load_data","repo":"IBM/transformers-struct-guidance","repo_kind":"official","path":"src/lm-sc.py","file_url":"https://github.com/IBM/transformers-struct-guidance/blob/HEAD/src/lm-sc.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b0e89cf46d9f15be"}},{"code_sha256_prefix":"57d95c99f22f1694","entry":"load_data","repo":"IBM/transformers-struct-guidance","repo_kind":"official","path":"src/lm.py","file_url":"https://github.com/IBM/transformers-struct-guidance/blob/HEAD/src/lm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"57d95c99f22f1694"}},{"code_sha256_prefix":"66c424b47ad705f6","entry":"load_sents","repo":"IBM/transformers-struct-guidance","repo_kind":"official","path":"src/lm-sc.py","file_url":"https://github.com/IBM/transformers-struct-guidance/blob/HEAD/src/lm-sc.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"66c424b47ad705f6"}},{"code_sha256_prefix":"6104f9390f126155","entry":"sample_from_scores","repo":"IBM/transformers-struct-guidance","repo_kind":"official","path":"src/plm-gen.py","file_url":"https://github.com/IBM/transformers-struct-guidance/blob/HEAD/src/plm-gen.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6104f9390f126155"}},{"code_sha256_prefix":"c17e76a84f6a0453","entry":"update_ngram_freq_dict","repo":"IBM/transformers-struct-guidance","repo_kind":"official","path":"src/get_action_ngram_list.py","file_url":"https://github.com/IBM/transformers-struct-guidance/blob/HEAD/src/get_action_ngram_list.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c17e76a84f6a0453"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}