{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/190600346","title":"Pre-training of Graph Augmented Transformers for Medication Recommendation","arxiv_id":"1906.00346","date":"2019-06-02","proceeding":null,"authors":["Junyuan Shang","Tengfei Ma","Cao Xiao","Jimeng Sun"],"abstract":"Medication recommendation is an important healthcare application. It is commonly formulated as a temporal prediction task. Hence, most existing works only utilize longitudinal electronic health records (EHRs) from a small number of patients with multiple visits ignoring a large number of patients with a single visit (selection bias). Moreover, important hierarchical knowledge such as diagnosis hierarchy is not leveraged in the representation learning process. To address these challenges, we propose G-BERT, a new model to combine the power of Graph Neural Networks (GNNs) and BERT (Bidirectional Encoder Representations from Transformers) for medical code representation and medication recommendation. We use GNNs to represent the internal hierarchical structures of medical codes. Then we integrate the GNN representation into a transformer-based visit encoder and pre-train it on EHR data from patients only with a single visit. The pre-trained visit encoder and representation are then fine-tuned for downstream predictive tasks on longitudinal EHRs from patients with multiple visits. G-BERT is the first to bring the language model pre-training schema into the healthcare domain and it achieved state-of-the-art performance on the medication recommendation task.","url_abs":"https://arxiv.org/abs/1906.00346v2","url_pdf":"https://arxiv.org/pdf/1906.00346v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"190600346","repo_url":"https://github.com/jshang123/G-Bert","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"representation-learning","task_name":"Representation Learning"},{"task_slug":"selection-bias","task_name":"Selection bias"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1906.00346","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1906.00346"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jshang123/G-Bert","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":1,"ran":1,"unverified":7},"by_repo_kind":{"official":{"samples":9,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"8d23fbe2b99b840b","entry":"gelu","repo":"jshang123/G-Bert","repo_kind":"official","path":"code/bert_models.py","file_url":"https://github.com/jshang123/G-Bert/blob/HEAD/code/bert_models.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8d23fbe2b99b840b"}},{"code_sha256_prefix":"2c4438c79bd4a86a","entry":"t2n","repo":"jshang123/G-Bert","repo_kind":"official","path":"code/utils.py","file_url":"https://github.com/jshang123/G-Bert/blob/HEAD/code/utils.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2c4438c79bd4a86a"}},{"code_sha256_prefix":"7be7468bc7867a06","entry":"build_cominbed_edges","repo":"jshang123/G-Bert","repo_kind":"official","path":"code/build_tree.py","file_url":"https://github.com/jshang123/G-Bert/blob/HEAD/code/build_tree.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7be7468bc7867a06"}},{"code_sha256_prefix":"12cb637233775372","entry":"build_stage_one_edges","repo":"jshang123/G-Bert","repo_kind":"official","path":"code/build_tree.py","file_url":"https://github.com/jshang123/G-Bert/blob/HEAD/code/build_tree.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"12cb637233775372"}},{"code_sha256_prefix":"c96d9ce1076a16a6","entry":"build_stage_two_edges","repo":"jshang123/G-Bert","repo_kind":"official","path":"code/build_tree.py","file_url":"https://github.com/jshang123/G-Bert/blob/HEAD/code/build_tree.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c96d9ce1076a16a6"}},{"code_sha256_prefix":"feaaa7cf4f1fc582","entry":"load_dataset","repo":"jshang123/G-Bert","repo_kind":"official","path":"code/run_gbert.py","file_url":"https://github.com/jshang123/G-Bert/blob/HEAD/code/run_gbert.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"feaaa7cf4f1fc582"}},{"code_sha256_prefix":"4d071c499ef55822","entry":"load_dataset","repo":"jshang123/G-Bert","repo_kind":"official","path":"code/run_gbert_side.py","file_url":"https://github.com/jshang123/G-Bert/blob/HEAD/code/run_gbert_side.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4d071c499ef55822"}},{"code_sha256_prefix":"8d954c8ce60d4edf","entry":"load_dataset","repo":"jshang123/G-Bert","repo_kind":"official","path":"code/run_pretraining.py","file_url":"https://github.com/jshang123/G-Bert/blob/HEAD/code/run_pretraining.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8d954c8ce60d4edf"}},{"code_sha256_prefix":"9166c871f0af6e16","entry":"random_word","repo":"jshang123/G-Bert","repo_kind":"official","path":"code/run_pretraining.py","file_url":"https://github.com/jshang123/G-Bert/blob/HEAD/code/run_pretraining.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9166c871f0af6e16"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}