{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/kepler-a-unified-model-for-knowledge","title":"KEPLER: A Unified Model for Knowledge Embedding and Pre-trained Language Representation","arxiv_id":"1911.06136","date":"2019-11-13","proceeding":null,"authors":["Xiaozhi Wang","Tianyu Gao","Zhaocheng Zhu","Zhengyan Zhang","Zhiyuan Liu","Juanzi Li","Jian Tang"],"abstract":"Pre-trained language representation models (PLMs) cannot well capture factual knowledge from text. In contrast, knowledge embedding (KE) methods can effectively represent the relational facts in knowledge graphs (KGs) with informative entity embeddings, but conventional KE models cannot take full advantage of the abundant textual information. In this paper, we propose a unified model for Knowledge Embedding and Pre-trained LanguagE Representation (KEPLER), which can not only better integrate factual knowledge into PLMs but also produce effective text-enhanced KE with the strong PLMs. In KEPLER, we encode textual entity descriptions with a PLM as their embeddings, and then jointly optimize the KE and language modeling objectives. Experimental results show that KEPLER achieves state-of-the-art performances on various NLP tasks, and also works remarkably well as an inductive KE model on KG link prediction. Furthermore, for pre-training and evaluating KEPLER, we construct Wikidata5M, a large-scale KG dataset with aligned entity descriptions, and benchmark state-of-the-art KE methods on it. It shall serve as a new KE benchmark and facilitate the research on large KG, inductive KE, and KG with text. The source code can be obtained from https://github.com/THU-KEG/KEPLER.","url_abs":"https://arxiv.org/abs/1911.06136v3","url_pdf":"https://arxiv.org/pdf/1911.06136v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"kepler-a-unified-model-for-knowledge","repo_url":"https://github.com/THU-KEG/KEPLER","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"entity-embeddings","task_name":"Entity Embeddings"},{"task_slug":"entity-typing","task_name":"Entity Typing"},{"task_slug":"inductive-knowledge-graph-completion","task_name":"Inductive knowledge graph completion"},{"task_slug":"knowledge-graph-completion","task_name":"Knowledge Graph Completion"},{"task_slug":"knowledge-graph-embeddings","task_name":"Knowledge Graph Embeddings"},{"task_slug":"knowledge-graphs","task_name":"Knowledge Graphs"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"link-prediction","task_name":"Link Prediction"},{"task_slug":"relation-classification","task_name":"Relation Classification"},{"task_slug":"relation-extraction","task_name":"Relation Extraction"}],"methods":[],"datasets_introduced":[{"slug":"wikidata5m","name":"Wikidata5M","full_name":""}],"methods_introduced":[],"results":[{"leaderboard":"/sota/inductive-knowledge-graph-completion-on-1","task":"Inductive knowledge graph completion","dataset":"Wikidata5m-ind","model":"KEPLER-Wiki-rel","rank_in_archive_order":2,"of":3,"metrics":{"Hits@1":"0.222","Hits@10":"0.73","Hits@3":"0.514","MRR":"0.402"},"uses_additional_data":false},{"leaderboard":"/sota/link-prediction-on-wikidata5m","task":"Link Prediction","dataset":"Wikidata5M","model":"SimplE","rank_in_archive_order":9,"of":14,"metrics":{"Hits@1":"0.252","Hits@10":"0.377","Hits@3":"0.317","MRR":"0.296"},"uses_additional_data":false},{"leaderboard":"/sota/link-prediction-on-wikidata5m","task":"Link Prediction","dataset":"Wikidata5M","model":"RotatE","rank_in_archive_order":10,"of":14,"metrics":{"Hits@1":"0.234","Hits@10":"0.39","Hits@3":"0.322","MRR":"0.29"},"uses_additional_data":false},{"leaderboard":"/sota/link-prediction-on-wikidata5m","task":"Link Prediction","dataset":"Wikidata5M","model":"ComplEx","rank_in_archive_order":11,"of":14,"metrics":{"Hits@1":"0.228","Hits@10":"0.373","Hits@3":"0.310","MRR":"0.281"},"uses_additional_data":false},{"leaderboard":"/sota/link-prediction-on-wikidata5m","task":"Link Prediction","dataset":"Wikidata5M","model":"TransE","rank_in_archive_order":12,"of":14,"metrics":{"Hits@1":"0.17","Hits@10":"0.392","Hits@3":"0.311","MRR":"0.253"},"uses_additional_data":false},{"leaderboard":"/sota/link-prediction-on-wikidata5m","task":"Link Prediction","dataset":"Wikidata5M","model":"DistMult","rank_in_archive_order":13,"of":14,"metrics":{"Hits@1":"0.208","Hits@10":"0.334","Hits@3":"0.278","MRR":"0.253"},"uses_additional_data":false},{"leaderboard":"/sota/link-prediction-on-wikidata5m","task":"Link Prediction","dataset":"Wikidata5M","model":"KEPLER-Wiki-rel","rank_in_archive_order":14,"of":14,"metrics":{"Hits@1":"0.173","Hits@10":"0.277","Hits@3":"0.224","MRR":"0.210"},"uses_additional_data":false},{"leaderboard":"/sota/relation-classification-on-tacred-1","task":"Relation Classification","dataset":"TACRED","model":"KEPLER","rank_in_archive_order":11,"of":17,"metrics":{"F1":"71.7"},"uses_additional_data":false},{"leaderboard":"/sota/relation-extraction-on-tacred","task":"Relation Extraction","dataset":"TACRED","model":"KEPLER","rank_in_archive_order":16,"of":40,"metrics":{"F1":"71.7"},"uses_additional_data":true}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1911.06136","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1911.06136"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/THU-KEG/KEPLER","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":1,"unverified":4},"by_repo_kind":{"official":{"samples":5,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":5,"samples":[{"code_sha256_prefix":"82bdf940665aeeb4","entry":"safe_readline","repo":"THU-KEG/KEPLER","repo_kind":"official","path":"fairseq/binarizer.py","file_url":"https://github.com/THU-KEG/KEPLER/blob/HEAD/fairseq/binarizer.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"82bdf940665aeeb4"}},{"code_sha256_prefix":"9eda7f80331d658c","entry":"dataset_dest_prefix","repo":"THU-KEG/KEPLER","repo_kind":"official","path":"preprocess.py","file_url":"https://github.com/THU-KEG/KEPLER/blob/HEAD/preprocess.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"9eda7f80331d658c"}},{"code_sha256_prefix":"c5ea9d48b6fc2167","entry":"load_checkpoint","repo":"THU-KEG/KEPLER","repo_kind":"official","path":"fairseq/checkpoint_utils.py","file_url":"https://github.com/THU-KEG/KEPLER/blob/HEAD/fairseq/checkpoint_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"c5ea9d48b6fc2167"}},{"code_sha256_prefix":"df40d96cc20e7f57","entry":"load_checkpoint_to_cpu","repo":"THU-KEG/KEPLER","repo_kind":"official","path":"fairseq/checkpoint_utils.py","file_url":"https://github.com/THU-KEG/KEPLER/blob/HEAD/fairseq/checkpoint_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"df40d96cc20e7f57"}},{"code_sha256_prefix":"b3616a62387703cb","entry":"save_checkpoint","repo":"THU-KEG/KEPLER","repo_kind":"official","path":"fairseq/checkpoint_utils.py","file_url":"https://github.com/THU-KEG/KEPLER/blob/HEAD/fairseq/checkpoint_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"b3616a62387703cb"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}