{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/semantics-aware-bert-for-language","title":"Semantics-aware BERT for Language Understanding","arxiv_id":"1909.02209","date":"2019-09-05","proceeding":null,"authors":["Zhuosheng Zhang","Yuwei Wu","Hai Zhao","Zuchao Li","Shuailiang Zhang","Xi Zhou","Xiang Zhou"],"abstract":"The latest work on language representations carefully integrates contextualized features into language model training, which enables a series of success especially in various machine reading comprehension and natural language inference tasks. However, the existing language representation models including ELMo, GPT and BERT only exploit plain context-sensitive features such as character or word embeddings. They rarely consider incorporating structured semantic information which can provide rich semantics for language representation. To promote natural language understanding, we propose to incorporate explicit contextual semantics from pre-trained semantic role labeling, and introduce an improved language representation model, Semantics-aware BERT (SemBERT), which is capable of explicitly absorbing contextual semantics over a BERT backbone. SemBERT keeps the convenient usability of its BERT precursor in a light fine-tuning way without substantial task-specific modifications. Compared with BERT, semantics-aware BERT is as simple in concept but more powerful. It obtains new state-of-the-art or substantially improves results on ten reading comprehension and language inference tasks.","url_abs":"https://arxiv.org/abs/1909.02209v3","url_pdf":"https://arxiv.org/pdf/1909.02209v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"semantics-aware-bert-for-language","repo_url":"https://github.com/cooelf/SemBERT","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"machine-reading-comprehension","task_name":"Machine Reading Comprehension"},{"task_slug":"natural-language-inference","task_name":"Natural Language Inference"},{"task_slug":"natural-language-understanding","task_name":"Natural Language Understanding"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"reading-comprehension","task_name":"Reading Comprehension"},{"task_slug":"semantic-role-labeling","task_name":"Semantic Role Labeling"},{"task_slug":"word-embeddings","task_name":"Word Embeddings"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"bilstm","method_name":"BiLSTM"},{"method_slug":"cosine-annealing","method_name":"Cosine Annealing"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"discriminative-fine-tuning","method_name":"Discriminative Fine-Tuning"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"elmo","method_name":"ELMo"},{"method_slug":"gpt","method_name":"GPT"},{"method_slug":"lstm","method_name":"LSTM"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-cosine-annealing","method_name":"Linear Warmup With Cosine Annealing"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"sigmoid-activation","method_name":"Sigmoid Activation"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"tanh-activation","method_name":"Tanh Activation"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/natural-language-inference-on-snli","task":"Natural Language Inference","dataset":"SNLI","model":"SemBERT","rank_in_archive_order":8,"of":98,"metrics":{"% Test Accuracy":"91.9","% Train Accuracy":"94.4","Parameters":"339m"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-squad20","task":"Question Answering","dataset":"SQuAD2.0","model":"SemBERT(ensemble)","rank_in_archive_order":106,"of":286,"metrics":{"EM":"86.166","F1":"88.886"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-squad20","task":"Question Answering","dataset":"SQuAD2.0","model":"SemBERT (ensemble)","rank_in_archive_order":107,"of":286,"metrics":{"EM":"86.166","F1":"88.886"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-squad20","task":"Question Answering","dataset":"SQuAD2.0","model":"SemBERT (single model)","rank_in_archive_order":132,"of":286,"metrics":{"EM":"84.800","F1":"87.864"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-squad20-dev","task":"Question Answering","dataset":"SQuAD2.0 dev","model":"SemBERT large","rank_in_archive_order":8,"of":13,"metrics":{"EM":"80.9","F1":"83.6"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1909.02209","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1909.02209"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/cooelf/SemBERT","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":2,"ran_honours":2,"unverified":8},"by_repo_kind":{"official":{"samples":12,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"0f786c407fb1ee4c","entry":"swish","repo":"cooelf/SemBERT","repo_kind":"official","path":"pytorch_pretrained_bert/modeling.py","file_url":"https://github.com/cooelf/SemBERT/blob/HEAD/pytorch_pretrained_bert/modeling.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":2,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0f786c407fb1ee4c"}},{"code_sha256_prefix":"eb725d5794b15f6b","entry":"accuracy","repo":"cooelf/SemBERT","repo_kind":"official","path":"run_scorer.py","file_url":"https://github.com/cooelf/SemBERT/blob/HEAD/run_scorer.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"eb725d5794b15f6b"}},{"code_sha256_prefix":"fdc64f4c72036ae4","entry":"gelu","repo":"cooelf/SemBERT","repo_kind":"official","path":"pytorch_pretrained_bert/modeling.py","file_url":"https://github.com/cooelf/SemBERT/blob/HEAD/pytorch_pretrained_bert/modeling.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fdc64f4c72036ae4"}},{"code_sha256_prefix":"5eff22fa0a651276","entry":"url_to_filename","repo":"cooelf/SemBERT","repo_kind":"official","path":"pytorch_pretrained_bert/file_utils.py","file_url":"https://github.com/cooelf/SemBERT/blob/HEAD/pytorch_pretrained_bert/file_utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5eff22fa0a651276"}},{"code_sha256_prefix":"df9f10631fa5d322","entry":"cached_path","repo":"cooelf/SemBERT","repo_kind":"official","path":"pytorch_pretrained_bert/file_utils.py","file_url":"https://github.com/cooelf/SemBERT/blob/HEAD/pytorch_pretrained_bert/file_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"df9f10631fa5d322"}},{"code_sha256_prefix":"517a71720243fbf7","entry":"detokenize","repo":"cooelf/SemBERT","repo_kind":"official","path":"data_process/util.py","file_url":"https://github.com/cooelf/SemBERT/blob/HEAD/data_process/util.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"517a71720243fbf7"}},{"code_sha256_prefix":"6db16fe8f67e56b6","entry":"filename_to_url","repo":"cooelf/SemBERT","repo_kind":"official","path":"pytorch_pretrained_bert/file_utils.py","file_url":"https://github.com/cooelf/SemBERT/blob/HEAD/pytorch_pretrained_bert/file_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6db16fe8f67e56b6"}},{"code_sha256_prefix":"99e01374e3f7e24f","entry":"get_tags","repo":"cooelf/SemBERT","repo_kind":"official","path":"tag_model/tagging.py","file_url":"https://github.com/cooelf/SemBERT/blob/HEAD/tag_model/tagging.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"99e01374e3f7e24f"}},{"code_sha256_prefix":"ebe8bc8fe2ec06d1","entry":"load_tag_vocab","repo":"cooelf/SemBERT","repo_kind":"official","path":"tag_model/tag_tokenization.py","file_url":"https://github.com/cooelf/SemBERT/blob/HEAD/tag_model/tag_tokenization.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ebe8bc8fe2ec06d1"}},{"code_sha256_prefix":"baa5766f4566aafc","entry":"load_tf_weights_in_bert","repo":"cooelf/SemBERT","repo_kind":"official","path":"pytorch_pretrained_bert/modeling.py","file_url":"https://github.com/cooelf/SemBERT/blob/HEAD/pytorch_pretrained_bert/modeling.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"baa5766f4566aafc"}},{"code_sha256_prefix":"5101ed51dc95ac5e","entry":"transform_tag_features","repo":"cooelf/SemBERT","repo_kind":"official","path":"run_scorer.py","file_url":"https://github.com/cooelf/SemBERT/blob/HEAD/run_scorer.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5101ed51dc95ac5e"}},{"code_sha256_prefix":"bdddb1a2d67bc868","entry":"transform_tag_features","repo":"cooelf/SemBERT","repo_kind":"official","path":"run_classifier.py","file_url":"https://github.com/cooelf/SemBERT/blob/HEAD/run_classifier.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bdddb1a2d67bc868"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}