{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2605-15562","title":"GiLT: Augmenting Transformer Language Models with Dependency Graphs","arxiv_id":"2605.15562","date":"2026-05-15","proceeding":null,"authors":["Tianyu Huang","Yida Zhao","Chuyan Zhou","Kewei Tu"],"abstract":"Augmenting Transformers with linguistic structures effectively enhances the syntactic generalization performance of language models. Previous work in this direction focuses on syntactic tree structures of languages, in particular constituency tree structures. We propose Graph-Infused Layers Transformer Language Model (GiLT) which leverages dependency graphs for augmenting Transformer language models. Unlike most previous work, GiLT does not insert extra structural tokens in language modeling; instead, it injects structural information into language modeling by modulating attention weights in the Transformer with features extracted from the dependency graph that is incrementally constructed along with token prediction. In our experiments, GiLT with semantic dependency graphs achieves better syntactic generalization while maintaining competitive perplexity in comparison with Transformer language model baselines. In addition, GiLT can be finetuned from a pretrained language model to achieve improved downstream task performance. Our code is released at https://github.com/cookie-pie-oops/GiLT-LM.","url_abs":"https://arxiv.org/abs/2605.15562","url_pdf":"https://arxiv.org/pdf/2605.15562","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2605.15562","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2605.15562"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/cookie-pie-oops/GiLT-LM","reach":null}],"summary":{"ran":3,"ran_honours":1,"ran_draft_wrong":1,"unverified":4},"by_repo_kind":{"found_in_text":{"samples":9,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":9,"samples":[{"code_sha256_prefix":"e0fa815ddb4f7161","entry":"PositionalEmbedding","repo":"cookie-pie-oops/GiLT-LM","repo_kind":"found_in_text","path":"src/model_bllip_dep.py","file_url":"https://github.com/cookie-pie-oops/GiLT-LM/blob/HEAD/src/model_bllip_dep.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e0fa815ddb4f7161"}},{"code_sha256_prefix":"3ee176bd4691d3fa","entry":"PositionwiseFF","repo":"cookie-pie-oops/GiLT-LM","repo_kind":"found_in_text","path":"src/model_bllip_dep.py","file_url":"https://github.com/cookie-pie-oops/GiLT-LM/blob/HEAD/src/model_bllip_dep.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3ee176bd4691d3fa"}},{"code_sha256_prefix":"d5d7f281cec21e15","entry":"calculate_depth","repo":"cookie-pie-oops/GiLT-LM","repo_kind":"found_in_text","path":"src/model_bllip_dep.py","file_url":"https://github.com/cookie-pie-oops/GiLT-LM/blob/HEAD/src/model_bllip_dep.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d5d7f281cec21e15"}},{"code_sha256_prefix":"6ac774f21bb49f51","entry":"dijkstra_heap","repo":"cookie-pie-oops/GiLT-LM","repo_kind":"found_in_text","path":"src/model_bllip_dep.py","file_url":"https://github.com/cookie-pie-oops/GiLT-LM/blob/HEAD/src/model_bllip_dep.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6ac774f21bb49f51"}},{"code_sha256_prefix":"f7ee1307a82993ec","entry":"find_label_idx","repo":"cookie-pie-oops/GiLT-LM","repo_kind":"found_in_text","path":"src/model_bllip_dep.py","file_url":"https://github.com/cookie-pie-oops/GiLT-LM/blob/HEAD/src/model_bllip_dep.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f7ee1307a82993ec"}},{"code_sha256_prefix":"c63fb381a3d18110","entry":"RelMultiHeadAttn","repo":"cookie-pie-oops/GiLT-LM","repo_kind":"found_in_text","path":"src/model_bllip_dep.py","file_url":"https://github.com/cookie-pie-oops/GiLT-LM/blob/HEAD/src/model_bllip_dep.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c63fb381a3d18110"}},{"code_sha256_prefix":"3b7d8fabefd13396","entry":"RelPartialLearnableMultiHeadAttn","repo":"cookie-pie-oops/GiLT-LM","repo_kind":"found_in_text","path":"src/model_bllip_dep.py","file_url":"https://github.com/cookie-pie-oops/GiLT-LM/blob/HEAD/src/model_bllip_dep.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3b7d8fabefd13396"}},{"code_sha256_prefix":"b81f189335a98635","entry":"TransformerGrammar","repo":"cookie-pie-oops/GiLT-LM","repo_kind":"found_in_text","path":"src/model_bllip_dep.py","file_url":"https://github.com/cookie-pie-oops/GiLT-LM/blob/HEAD/src/model_bllip_dep.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b81f189335a98635"}},{"code_sha256_prefix":"df6e87e797e6a7c2","entry":"TransformerGrammarLayer","repo":"cookie-pie-oops/GiLT-LM","repo_kind":"found_in_text","path":"src/model_bllip_dep.py","file_url":"https://github.com/cookie-pie-oops/GiLT-LM/blob/HEAD/src/model_bllip_dep.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"df6e87e797e6a7c2"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.CL","source":"arxiv_2026.jsonl"},"syntology_extracted_results":null}