{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/luke-deep-contextualized-entity","title":"LUKE: Deep Contextualized Entity Representations with Entity-aware Self-attention","arxiv_id":"2010.01057","date":"2020-10-02","proceeding":"EMNLP 2020 11","authors":["Ikuya Yamada","Akari Asai","Hiroyuki Shindo","Hideaki Takeda","Yuji Matsumoto"],"abstract":"Entity representations are useful in natural language tasks involving entities. In this paper, we propose new pretrained contextualized representations of words and entities based on the bidirectional transformer. The proposed model treats words and entities in a given text as independent tokens, and outputs contextualized representations of them. Our model is trained using a new pretraining task based on the masked language model of BERT. The task involves predicting randomly masked words and entities in a large entity-annotated corpus retrieved from Wikipedia. We also propose an entity-aware self-attention mechanism that is an extension of the self-attention mechanism of the transformer, and considers the types of tokens (words or entities) when computing attention scores. The proposed model achieves impressive empirical performance on a wide range of entity-related tasks. In particular, it obtains state-of-the-art results on five well-known datasets: Open Entity (entity typing), TACRED (relation classification), CoNLL-2003 (named entity recognition), ReCoRD (cloze-style question answering), and SQuAD 1.1 (extractive question answering). Our source code and pretrained representations are available at https://github.com/studio-ousia/luke.","url_abs":"https://arxiv.org/abs/2010.01057v1","url_pdf":"https://arxiv.org/pdf/2010.01057v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"luke-deep-contextualized-entity","repo_url":"https://github.com/studio-ousia/luke","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"luke-deep-contextualized-entity","repo_url":"https://github.com/JiachengLi1995/UCTopic","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"luke-deep-contextualized-entity","repo_url":"https://github.com/huggingface/transformers","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"luke-deep-contextualized-entity","repo_url":"https://github.com/shmulvad/zero-for-ner","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"unanswered"}},{"paper_slug":"luke-deep-contextualized-entity","repo_url":"https://github.com/2023-MindSpore-1/ms-code-156","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":{"status":"ok"}},{"paper_slug":"luke-deep-contextualized-entity","repo_url":"https://github.com/Beacontownfc/paddle_luke_stable","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"paddle","reach":{"status":"ok"}},{"paper_slug":"luke-deep-contextualized-entity","repo_url":"https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/language_model/roformer","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"paddle","reach":null},{"paper_slug":"luke-deep-contextualized-entity","repo_url":"https://github.com/mindspore-ai/models/tree/master/research/nlp/luke","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null},{"paper_slug":"luke-deep-contextualized-entity","repo_url":"https://github.com/pwc-1/Paper-9/tree/main/luke","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null}],"tasks":[{"task_slug":"common-sense-reasoning","task_name":"Common Sense Reasoning"},{"task_slug":"entity-typing","task_name":"Entity Typing"},{"task_slug":"extractive-question-answering","task_name":"Extractive Question-Answering"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"named-entity-recognition-1","task_name":"Named Entity Recognition"},{"task_slug":"named-entity-recognition-ner","task_name":"Named Entity Recognition (NER)"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"relation-classification","task_name":"Relation Classification"},{"task_slug":"relation-extraction","task_name":"Relation Extraction"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/common-sense-reasoning-on-record","task":"Common Sense Reasoning","dataset":"ReCoRD","model":"LUKE 483M","rank_in_archive_order":10,"of":45,"metrics":{"EM":"90.6","F1":"91.2"},"uses_additional_data":false},{"leaderboard":"/sota/entity-typing-on-open-entity-1","task":"Entity Typing","dataset":"Open Entity","model":"MLMET","rank_in_archive_order":1,"of":13,"metrics":{"F1":"78.2"},"uses_additional_data":false},{"leaderboard":"/sota/named-entity-recognition-ner-on-conll-2003","task":"Named Entity Recognition (NER)","dataset":"CoNLL 2003 (English)","model":"LUKE 483M","rank_in_archive_order":2,"of":73,"metrics":{"F1":"94.3"},"uses_additional_data":false},{"leaderboard":"/sota/named-entity-recognition-on-conll","task":"Named Entity Recognition (NER)","dataset":"CoNLL++","model":"LUKE（Large）","rank_in_archive_order":2,"of":11,"metrics":{"F1":"95.89"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-squad11","task":"Question Answering","dataset":"SQuAD1.1","model":"LUKE (single model)","rank_in_archive_order":3,"of":213,"metrics":{"EM":"90.202","F1":"95.379"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-squad11","task":"Question Answering","dataset":"SQuAD1.1","model":"LUKE","rank_in_archive_order":4,"of":213,"metrics":{"EM":"90.2"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-squad11","task":"Question Answering","dataset":"SQuAD1.1","model":"LUKE 483M","rank_in_archive_order":208,"of":213,"metrics":{"F1":"95.4"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-squad11-dev","task":"Question Answering","dataset":"SQuAD1.1 dev","model":"LUKE","rank_in_archive_order":2,"of":55,"metrics":{"EM":"89.8"},"uses_additional_data":true},{"leaderboard":"/sota/question-answering-on-squad11-dev","task":"Question Answering","dataset":"SQuAD1.1 dev","model":"LUKE 483M","rank_in_archive_order":51,"of":55,"metrics":{"F1":"95"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-squad20","task":"Question Answering","dataset":"SQuAD2.0","model":"LUKE (single model)","rank_in_archive_order":84,"of":286,"metrics":{"EM":"87.429","F1":"90.163"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-squad20","task":"Question Answering","dataset":"SQuAD2.0","model":"LUKE 483M","rank_in_archive_order":284,"of":286,"metrics":{"F1":"90.2"},"uses_additional_data":false},{"leaderboard":"/sota/relation-classification-on-tacred-1","task":"Relation Classification","dataset":"TACRED","model":"LUKE 483M","rank_in_archive_order":14,"of":17,"metrics":{"F1":"72.7"},"uses_additional_data":false},{"leaderboard":"/sota/relation-extraction-on-tacred","task":"Relation Extraction","dataset":"TACRED","model":"LUKE","rank_in_archive_order":40,"of":40,"metrics":{"F1 (1% Few-Shot)":"17.0","F1 (5% Few-Shot)":"51.6"},"uses_additional_data":true}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2010.01057","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.01057"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/huggingface/transformers","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Beacontownfc/paddle_luke_stable","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/PaddlePaddle/PaddleNLP/tree/develop/examples/language_model/roformer","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/shmulvad/zero-for-ner","reach":{"status":"unanswered"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mindspore-ai/models/tree/master/research/nlp/luke","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/pwc-1/Paper-9/tree/main/luke","reach":null},{"provenance":"deterministic:regex_extraction","url":"https://github.com/studio-ousia/luke","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/JiachengLi1995/UCTopic","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/2023-MindSpore-1/ms-code-156","reach":{"status":"ok"}}],"summary":{"ran":3,"unverified":7},"by_repo_kind":{"official":{"samples":8,"ran":3,"repositories":1},"listed":{"samples":2,"ran":0,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"fb78ccc0f682352e","entry":"EntityAwareAttention","repo":"studio-ousia/luke","repo_kind":"official","path":"luke/model.py","file_url":"https://github.com/studio-ousia/luke/blob/HEAD/luke/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fb78ccc0f682352e"}},{"code_sha256_prefix":"8d7fce1bd60c4918","entry":"EntityEmbeddings","repo":"studio-ousia/luke","repo_kind":"official","path":"luke/model.py","file_url":"https://github.com/studio-ousia/luke/blob/HEAD/luke/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8d7fce1bd60c4918"}},{"code_sha256_prefix":"65315a42df7afec6","entry":"LukeConfig","repo":"studio-ousia/luke","repo_kind":"official","path":"luke/model.py","file_url":"https://github.com/studio-ousia/luke/blob/HEAD/luke/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"65315a42df7afec6"}},{"code_sha256_prefix":"5f47aef368b5af00","entry":"EntityAwareEncoder","repo":"studio-ousia/luke","repo_kind":"official","path":"luke/model.py","file_url":"https://github.com/studio-ousia/luke/blob/HEAD/luke/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5f47aef368b5af00"}},{"code_sha256_prefix":"75c5d0233ea564ee","entry":"EntityAwareLayer","repo":"studio-ousia/luke","repo_kind":"official","path":"luke/model.py","file_url":"https://github.com/studio-ousia/luke/blob/HEAD/luke/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"75c5d0233ea564ee"}},{"code_sha256_prefix":"6139734334c69c5d","entry":"LukeEncoder","repo":"studio-ousia/luke","repo_kind":"official","path":"luke/model.py","file_url":"https://github.com/studio-ousia/luke/blob/HEAD/luke/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6139734334c69c5d"}},{"code_sha256_prefix":"de51e92ef6ec77d5","entry":"LukeEntityAwareAttentionModel","repo":"studio-ousia/luke","repo_kind":"official","path":"luke/model.py","file_url":"https://github.com/studio-ousia/luke/blob/HEAD/luke/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"de51e92ef6ec77d5"}},{"code_sha256_prefix":"607f66ef6841b94f","entry":"LukeModel","repo":"studio-ousia/luke","repo_kind":"official","path":"luke/model.py","file_url":"https://github.com/studio-ousia/luke/blob/HEAD/luke/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"607f66ef6841b94f"}},{"code_sha256_prefix":"12ca593ec040ed10","entry":"LukeSelfAttention","repo":"huggingface/transformers","repo_kind":"listed","path":"src/transformers/models/luke/modeling_luke.py","file_url":"https://github.com/huggingface/transformers/blob/HEAD/src/transformers/models/luke/modeling_luke.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"12ca593ec040ed10"}},{"code_sha256_prefix":"42a369215b401697","entry":"UCTopicModel","repo":"JiachengLi1995/UCTopic","repo_kind":"listed","path":"uctopic/models.py","file_url":"https://github.com/JiachengLi1995/UCTopic/blob/HEAD/uctopic/models.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"42a369215b401697"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}