{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/linkbert-pretraining-language-models-with","title":"LinkBERT: Pretraining Language Models with Document Links","arxiv_id":"2203.15827","date":"2022-03-29","proceeding":"ACL 2022 5","authors":["Michihiro Yasunaga","Jure Leskovec","Percy Liang"],"abstract":"Language model (LM) pretraining can learn various knowledge from text corpora, helping downstream tasks. However, existing methods such as BERT model a single document, and do not capture dependencies or knowledge that span across documents. In this work, we propose LinkBERT, an LM pretraining method that leverages links between documents, e.g., hyperlinks. Given a text corpus, we view it as a graph of documents and create LM inputs by placing linked documents in the same context. We then pretrain the LM with two joint self-supervised objectives: masked language modeling and our new proposal, document relation prediction. We show that LinkBERT outperforms BERT on various downstream tasks across two domains: the general domain (pretrained on Wikipedia with hyperlinks) and biomedical domain (pretrained on PubMed with citation links). LinkBERT is especially effective for multi-hop reasoning and few-shot QA (+5% absolute improvement on HotpotQA and TriviaQA), and our biomedical LinkBERT sets new states of the art on various BioNLP tasks (+7% on BioASQ and USMLE). We release our pretrained models, LinkBERT and BioLinkBERT, as well as code and data at https://github.com/michiyasunaga/LinkBERT.","url_abs":"https://arxiv.org/abs/2203.15827v1","url_pdf":"https://arxiv.org/pdf/2203.15827v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"linkbert-pretraining-language-models-with","repo_url":"https://github.com/michiyasunaga/LinkBERT","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"document-classification","task_name":"Document Classification"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"masked-language-modeling","task_name":"Masked Language Modeling"},{"task_slug":"medical-relation-extraction","task_name":"Medical Relation Extraction"},{"task_slug":"named-entity-recognition-ner","task_name":"Named Entity Recognition (NER)"},{"task_slug":"pico","task_name":"PICO"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"relation-extraction","task_name":"Relation Extraction"},{"task_slug":"relation-prediction","task_name":"Relation Prediction"},{"task_slug":"semantic-similarity","task_name":"Semantic Similarity"},{"task_slug":"sentence-similarity","task_name":"Sentence Similarity"},{"task_slug":"text-classification","task_name":"Text Classification"},{"task_slug":"triviaqa","task_name":"TriviaQA"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/document-classification-on-hoc","task":"Document Classification","dataset":"HOC","model":"BioLinkBERT (large)","rank_in_archive_order":1,"of":5,"metrics":{"F1":"88.1","Micro F1":"84.87"},"uses_additional_data":false},{"leaderboard":"/sota/medical-relation-extraction-on-ddi-extraction","task":"Medical Relation Extraction","dataset":"DDI extraction 2013 corpus","model":"BioLinkBERT (large)","rank_in_archive_order":1,"of":2,"metrics":{"F1":"83.35"},"uses_additional_data":false},{"leaderboard":"/sota/named-entity-recognition-on-bc2gm","task":"Named Entity Recognition (NER)","dataset":"BC2GM","model":"BioLinkBERT (large)","rank_in_archive_order":8,"of":13,"metrics":{"F1":"85.18"},"uses_additional_data":false},{"leaderboard":"/sota/named-entity-recognition-ner-on-bc5cdr","task":"Named Entity Recognition (NER)","dataset":"BC5CDR","model":"BioLinkBERT (large)","rank_in_archive_order":6,"of":16,"metrics":{"F1":"90.22"},"uses_additional_data":true},{"leaderboard":"/sota/named-entity-recognition-on-bc5cdr-chemical","task":"Named Entity Recognition (NER)","dataset":"BC5CDR-chemical","model":"BioLinkBERT (large)","rank_in_archive_order":9,"of":13,"metrics":{"F1":"94.04"},"uses_additional_data":false},{"leaderboard":"/sota/named-entity-recognition-on-bc5cdr-disease","task":"Named Entity Recognition (NER)","dataset":"BC5CDR-disease","model":"BioLinkBERT (large)","rank_in_archive_order":5,"of":10,"metrics":{"F1":"86.39"},"uses_additional_data":false},{"leaderboard":"/sota/named-entity-recognition-ner-on-jnlpba","task":"Named Entity Recognition (NER)","dataset":"JNLPBA","model":"BioLinkBERT (large)","rank_in_archive_order":6,"of":17,"metrics":{"F1":"80.06"},"uses_additional_data":false},{"leaderboard":"/sota/named-entity-recognition-ner-on-ncbi-disease","task":"Named Entity Recognition (NER)","dataset":"NCBI-disease","model":"BioLinkBERT (large)","rank_in_archive_order":9,"of":26,"metrics":{"F1":"88.76"},"uses_additional_data":false},{"leaderboard":"/sota/pico-on-ebm-pico","task":"PICO","dataset":"EBM PICO","model":"BioLinkBERT (large)","rank_in_archive_order":1,"of":3,"metrics":{"Macro F1 word level":"74.19"},"uses_additional_data":false},{"leaderboard":"/sota/pico-on-ebm-pico","task":"PICO","dataset":"EBM PICO","model":"BioLinkBERT (base)","rank_in_archive_order":2,"of":3,"metrics":{"Macro F1 word level":"73.97 "},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-blurb","task":"Question Answering","dataset":"BLURB","model":"BioLinkBERT (large)","rank_in_archive_order":1,"of":4,"metrics":{"Accuracy":"83.5"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-blurb","task":"Question Answering","dataset":"BLURB","model":"BioLinkBERT (base)","rank_in_archive_order":2,"of":4,"metrics":{"Accuracy":"80.81"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-bioasq","task":"Question Answering","dataset":"BioASQ","model":"BioLinkBERT (large)","rank_in_archive_order":1,"of":7,"metrics":{"Accuracy":"94.8"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-bioasq","task":"Question Answering","dataset":"BioASQ","model":"BioLinkBERT (base)","rank_in_archive_order":3,"of":7,"metrics":{"Accuracy":"91.4"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-mrqa-2019","task":"Question Answering","dataset":"MRQA","model":"LinkBERT (large)","rank_in_archive_order":1,"of":2,"metrics":{"Average F1":"81.0"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-medqa-usmle","task":"Question Answering","dataset":"MedQA","model":"BioLinkBERT (base)","rank_in_archive_order":21,"of":27,"metrics":{"Accuracy":"40.0"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-newsqa","task":"Question Answering","dataset":"NewsQA","model":"LinkBERT (large)","rank_in_archive_order":17,"of":18,"metrics":{"F1":"72.6"},"uses_additional_data":true},{"leaderboard":"/sota/question-answering-on-pubmedqa","task":"Question Answering","dataset":"PubMedQA","model":"BioLinkBERT (large)","rank_in_archive_order":22,"of":30,"metrics":{"Accuracy":"72.2"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-pubmedqa","task":"Question Answering","dataset":"PubMedQA","model":"BioLinkBERT (base)","rank_in_archive_order":23,"of":30,"metrics":{"Accuracy":"70.2"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-squad11","task":"Question Answering","dataset":"SQuAD1.1","model":"LinkBERT (large)","rank_in_archive_order":16,"of":213,"metrics":{"EM":"87.45","F1":"92.7"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-triviaqa","task":"Question Answering","dataset":"TriviaQA","model":"LinkBERT (large)","rank_in_archive_order":51,"of":56,"metrics":{"F1":"78.2"},"uses_additional_data":false},{"leaderboard":"/sota/relation-extraction-on-chemprot","task":"Relation Extraction","dataset":"ChemProt","model":"BioLinkBERT (large)","rank_in_archive_order":3,"of":13,"metrics":{"F1":"79.98","Micro F1":"79.98"},"uses_additional_data":false},{"leaderboard":"/sota/relation-extraction-on-ddi","task":"Relation Extraction","dataset":"DDI","model":"BioLinkBERT (large)","rank_in_archive_order":1,"of":3,"metrics":{"F1":"83.35","Micro F1":"83.35"},"uses_additional_data":false},{"leaderboard":"/sota/relation-extraction-on-gad","task":"Relation Extraction","dataset":"GAD","model":"BioLinkBERT (large)","rank_in_archive_order":1,"of":3,"metrics":{"F1":"84.90","Micro F1":"84.90"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-similarity-on-biosses","task":"Semantic Similarity","dataset":"BIOSSES","model":"BioLinkBERT (large)","rank_in_archive_order":1,"of":3,"metrics":{"Pearson Correlation":"0.9363"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-similarity-on-biosses","task":"Semantic Similarity","dataset":"BIOSSES","model":"BioLinkBERT (base)","rank_in_archive_order":2,"of":3,"metrics":{"Pearson Correlation":"0.9325"},"uses_additional_data":false},{"leaderboard":"/sota/text-classification-on-blurb","task":"Text Classification","dataset":"BLURB","model":"BioLinkBERT (large)","rank_in_archive_order":1,"of":3,"metrics":{"F1":"84.88"},"uses_additional_data":false},{"leaderboard":"/sota/text-classification-on-blurb","task":"Text Classification","dataset":"BLURB","model":"BioLinkBERT (base)","rank_in_archive_order":2,"of":3,"metrics":{"F1":"84.35"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2203.15827","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.15827"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/michiyasunaga/LinkBERT","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/attardi/wikiextractor","reach":{"status":"ok","spdx":"AGPL-3.0"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/titipata/pubmed_parser","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":14},"by_repo_kind":{"official":{"samples":5,"ran":0,"repositories":1},"found_in_text":{"samples":9,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"74040dc1e7877b49","entry":"compute_p_r_f","repo":"michiyasunaga/LinkBERT","repo_kind":"official","path":"src/seqcls/utils_hoc.py","file_url":"https://github.com/michiyasunaga/LinkBERT/blob/HEAD/src/seqcls/utils_hoc.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"74040dc1e7877b49"}},{"code_sha256_prefix":"c2cd56888edff13a","entry":"divide","repo":"michiyasunaga/LinkBERT","repo_kind":"official","path":"src/seqcls/utils_hoc.py","file_url":"https://github.com/michiyasunaga/LinkBERT/blob/HEAD/src/seqcls/utils_hoc.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c2cd56888edff13a"}},{"code_sha256_prefix":"e3f1559c8f5f8688","entry":"eval_hoc","repo":"michiyasunaga/LinkBERT","repo_kind":"official","path":"src/seqcls/utils_hoc.py","file_url":"https://github.com/michiyasunaga/LinkBERT/blob/HEAD/src/seqcls/utils_hoc.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e3f1559c8f5f8688"}},{"code_sha256_prefix":"bf61041fdd6647e6","entry":"flatten_zip_author","repo":"titipata/pubmed_parser","repo_kind":"found_in_text","path":"pubmed_parser/pubmed_oa_parser.py","file_url":"https://github.com/titipata/pubmed_parser/blob/HEAD/pubmed_parser/pubmed_oa_parser.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bf61041fdd6647e6"}},{"code_sha256_prefix":"9b111e4814825d40","entry":"list_xml_path","repo":"titipata/pubmed_parser","repo_kind":"found_in_text","path":"pubmed_parser/pubmed_oa_parser.py","file_url":"https://github.com/titipata/pubmed_parser/blob/HEAD/pubmed_parser/pubmed_oa_parser.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9b111e4814825d40"}},{"code_sha256_prefix":"8abb78363c80dab0","entry":"parse_doi","repo":"titipata/pubmed_parser","repo_kind":"found_in_text","path":"pubmed_parser/medline_parser.py","file_url":"https://github.com/titipata/pubmed_parser/blob/HEAD/pubmed_parser/medline_parser.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8abb78363c80dab0"}},{"code_sha256_prefix":"0d69b1b23dd6057d","entry":"parse_mesh_terms","repo":"titipata/pubmed_parser","repo_kind":"found_in_text","path":"pubmed_parser/medline_parser.py","file_url":"https://github.com/titipata/pubmed_parser/blob/HEAD/pubmed_parser/medline_parser.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0d69b1b23dd6057d"}},{"code_sha256_prefix":"0a88d93a2312fac0","entry":"parse_pmid","repo":"titipata/pubmed_parser","repo_kind":"found_in_text","path":"pubmed_parser/medline_parser.py","file_url":"https://github.com/titipata/pubmed_parser/blob/HEAD/pubmed_parser/medline_parser.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0a88d93a2312fac0"}},{"code_sha256_prefix":"5b766af778be529d","entry":"parse_pubmed_web_tree","repo":"titipata/pubmed_parser","repo_kind":"found_in_text","path":"pubmed_parser/pubmed_web_parser.py","file_url":"https://github.com/titipata/pubmed_parser/blob/HEAD/pubmed_parser/pubmed_web_parser.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5b766af778be529d"}},{"code_sha256_prefix":"3957d7cb42610910","entry":"postprocess_qa_predictions","repo":"michiyasunaga/LinkBERT","repo_kind":"official","path":"src/qa/utils_qa.py","file_url":"https://github.com/michiyasunaga/LinkBERT/blob/HEAD/src/qa/utils_qa.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3957d7cb42610910"}},{"code_sha256_prefix":"31453e36e3e54527","entry":"postprocess_qa_predictions_with_beam_search","repo":"michiyasunaga/LinkBERT","repo_kind":"official","path":"src/qa/utils_qa.py","file_url":"https://github.com/michiyasunaga/LinkBERT/blob/HEAD/src/qa/utils_qa.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"31453e36e3e54527"}},{"code_sha256_prefix":"3ad8ed067be5af35","entry":"stringify_affiliation","repo":"titipata/pubmed_parser","repo_kind":"found_in_text","path":"pubmed_parser/utils.py","file_url":"https://github.com/titipata/pubmed_parser/blob/HEAD/pubmed_parser/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3ad8ed067be5af35"}},{"code_sha256_prefix":"bec7a30d2cf6ff0d","entry":"stringify_children","repo":"titipata/pubmed_parser","repo_kind":"found_in_text","path":"pubmed_parser/utils.py","file_url":"https://github.com/titipata/pubmed_parser/blob/HEAD/pubmed_parser/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bec7a30d2cf6ff0d"}},{"code_sha256_prefix":"89ed942af19f2fac","entry":"zip_author","repo":"titipata/pubmed_parser","repo_kind":"found_in_text","path":"pubmed_parser/pubmed_oa_parser.py","file_url":"https://github.com/titipata/pubmed_parser/blob/HEAD/pubmed_parser/pubmed_oa_parser.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"89ed942af19f2fac"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}