{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/spanbert-improving-pre-training-by","title":"SpanBERT: Improving Pre-training by Representing and Predicting Spans","arxiv_id":"1907.10529","date":"2019-07-24","proceeding":"TACL 2020 1","authors":["Mandar Joshi","Danqi Chen","Yinhan Liu","Daniel S. Weld","Luke Zettlemoyer","Omer Levy"],"abstract":"We present SpanBERT, a pre-training method that is designed to better represent and predict spans of text. Our approach extends BERT by (1) masking contiguous random spans, rather than random tokens, and (2) training the span boundary representations to predict the entire content of the masked span, without relying on the individual token representations within it. SpanBERT consistently outperforms BERT and our better-tuned baselines, with substantial gains on span selection tasks such as question answering and coreference resolution. In particular, with the same training data and model size as BERT-large, our single model obtains 94.6% and 88.7% F1 on SQuAD 1.1 and 2.0, respectively. We also achieve a new state of the art on the OntoNotes coreference resolution task (79.6\\% F1), strong performance on the TACRED relation extraction benchmark, and even show gains on GLUE.","url_abs":"https://arxiv.org/abs/1907.10529v3","url_pdf":"https://arxiv.org/pdf/1907.10529v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"spanbert-improving-pre-training-by","repo_url":"https://github.com/facebookresearch/SpanBERT","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"spanbert-improving-pre-training-by","repo_url":"https://github.com/UnknownGenie/altered-BERT-KPE","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"spanbert-improving-pre-training-by","repo_url":"https://github.com/amore-upf/masked-coreference","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"spanbert-improving-pre-training-by","repo_url":"https://github.com/mandarjoshi90/coref","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"spanbert-improving-pre-training-by","repo_url":"https://github.com/wooseok-AI/Korean_e2e_CR_BERT","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"spanbert-improving-pre-training-by","repo_url":"https://github.com/zixinzeng-jennifer/spanbert_trans","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}}],"tasks":[{"task_slug":"coreference-resolution","task_name":"Coreference Resolution"},{"task_slug":"linguistic-acceptability","task_name":"Linguistic Acceptability"},{"task_slug":"natural-language-inference","task_name":"Natural Language Inference"},{"task_slug":"open-domain-question-answering","task_name":"Open-Domain Question Answering"},{"task_slug":"paraphrase-identification","task_name":"Paraphrase Identification"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"relation-classification","task_name":"Relation Classification"},{"task_slug":"relation-extraction","task_name":"Relation Extraction"},{"task_slug":"semantic-textual-similarity","task_name":"Semantic Textual Similarity"},{"task_slug":"sentiment-analysis","task_name":"Sentiment Analysis"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/coreference-resolution-on-ontonotes","task":"Coreference Resolution","dataset":"OntoNotes","model":"SpanBERT","rank_in_archive_order":11,"of":26,"metrics":{"F1":"79.6"},"uses_additional_data":false},{"leaderboard":"/sota/linguistic-acceptability-on-cola","task":"Linguistic Acceptability","dataset":"CoLA","model":"SpanBERT","rank_in_archive_order":25,"of":43,"metrics":{"Accuracy":"64.3%"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-multinli","task":"Natural Language Inference","dataset":"MultiNLI","model":"SpanBERT","rank_in_archive_order":16,"of":67,"metrics":{"Matched":"88.1"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-qnli","task":"Natural Language Inference","dataset":"QNLI","model":"SpanBERT","rank_in_archive_order":17,"of":43,"metrics":{"Accuracy":"94.3%"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-rte","task":"Natural Language Inference","dataset":"RTE","model":"SpanBERT","rank_in_archive_order":40,"of":90,"metrics":{"Accuracy":"79.0%"},"uses_additional_data":false},{"leaderboard":"/sota/open-domain-question-answering-on-searchqa","task":"Open-Domain Question Answering","dataset":"SearchQA","model":"SpanBERT","rank_in_archive_order":14,"of":14,"metrics":{"F1":"84.8"},"uses_additional_data":false},{"leaderboard":"/sota/paraphrase-identification-on-quora-question","task":"Paraphrase Identification","dataset":"Quora Question Pairs","model":"SpanBERT","rank_in_archive_order":13,"of":31,"metrics":{"Accuracy":"89.5","F1":"71.9"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-naturalqa","task":"Question Answering","dataset":"NaturalQA","model":"SpanBERT","rank_in_archive_order":3,"of":4,"metrics":{"F1":"82.5"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-newsqa","task":"Question Answering","dataset":"NewsQA","model":"SpanBERT","rank_in_archive_order":16,"of":18,"metrics":{"F1":"73.6"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-squad11","task":"Question Answering","dataset":"SQuAD1.1","model":"SpanBERT (single model)","rank_in_archive_order":13,"of":213,"metrics":{"EM":"88.8","F1":"94.6","Hardware Burden":"586G"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-squad20","task":"Question Answering","dataset":"SQuAD2.0","model":"SpanBERT","rank_in_archive_order":120,"of":286,"metrics":{"EM":"85.7","F1":"88.7"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-squad20-dev","task":"Question Answering","dataset":"SQuAD2.0 dev","model":"SpanBERT","rank_in_archive_order":6,"of":13,"metrics":{"F1":"86.8"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-triviaqa","task":"Question Answering","dataset":"TriviaQA","model":"SpanBERT","rank_in_archive_order":48,"of":56,"metrics":{"F1":"83.6"},"uses_additional_data":false},{"leaderboard":"/sota/relation-classification-on-tacred-1","task":"Relation Classification","dataset":"TACRED","model":"SpanBERT","rank_in_archive_order":7,"of":17,"metrics":{"F1":"70.8"},"uses_additional_data":false},{"leaderboard":"/sota/relation-extraction-on-re-tacred","task":"Relation Extraction","dataset":"Re-TACRED","model":"SpanBERT","rank_in_archive_order":5,"of":9,"metrics":{"F1":"85.3"},"uses_additional_data":false},{"leaderboard":"/sota/relation-extraction-on-tacred","task":"Relation Extraction","dataset":"TACRED","model":"SpanBERT-large","rank_in_archive_order":22,"of":40,"metrics":{"F1":"70.8"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-mrpc","task":"Semantic Textual Similarity","dataset":"MRPC","model":"SpanBERT","rank_in_archive_order":8,"of":45,"metrics":{"Accuracy":"90.9%"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-sts-benchmark","task":"Semantic Textual Similarity","dataset":"STS Benchmark","model":"SpanBERT","rank_in_archive_order":21,"of":66,"metrics":{"Pearson Correlation":"0.899"},"uses_additional_data":false},{"leaderboard":"/sota/sentiment-analysis-on-sst-2-binary","task":"Sentiment Analysis","dataset":"SST-2 Binary classification","model":"SpanBERT","rank_in_archive_order":29,"of":87,"metrics":{"Accuracy":"94.8"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1907.10529","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1907.10529"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/UnknownGenie/altered-BERT-KPE","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/wooseok-AI/Korean_e2e_CR_BERT","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/SpanBERT","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/zixinzeng-jennifer/spanbert_trans","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mandarjoshi90/coref","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/amore-upf/masked-coreference","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_draft_wrong":3,"unverified":12},"by_repo_kind":{"official":{"samples":3,"ran":2,"repositories":1},"listed":{"samples":11,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":4,"samples":[{"code_sha256_prefix":"3860f4e19d89be73","entry":"convert_examples_to_features","repo":"facebookresearch/SpanBERT","repo_kind":"official","path":"code/run_tacred.py","file_url":"https://github.com/facebookresearch/SpanBERT/blob/HEAD/code/run_tacred.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"3860f4e19d89be73"}},{"code_sha256_prefix":"1731b0e57e413f9d","entry":"convert_token","repo":"facebookresearch/SpanBERT","repo_kind":"official","path":"code/run_tacred.py","file_url":"https://github.com/facebookresearch/SpanBERT/blob/HEAD/code/run_tacred.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"1731b0e57e413f9d"}},{"code_sha256_prefix":"5681f7dd9b6e3679","entry":"get_activation","repo":"amore-upf/masked-coreference","repo_kind":"listed","path":"bert/modeling.py","file_url":"https://github.com/amore-upf/masked-coreference/blob/HEAD/bert/modeling.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5681f7dd9b6e3679"}},{"code_sha256_prefix":"599218961a8155ee","entry":"b_cubed","repo":"amore-upf/masked-coreference","repo_kind":"listed","path":"metrics.py","file_url":"https://github.com/amore-upf/masked-coreference/blob/HEAD/metrics.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"599218961a8155ee"}},{"code_sha256_prefix":"da5d9e9f738be2ba","entry":"convert_examples_to_features","repo":"facebookresearch/SpanBERT","repo_kind":"official","path":"code/run_squad.py","file_url":"https://github.com/facebookresearch/SpanBERT/blob/HEAD/code/run_squad.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"da5d9e9f738be2ba"}},{"code_sha256_prefix":"ea92f536bea4cbd4","entry":"evaluate_conll","repo":"amore-upf/masked-coreference","repo_kind":"listed","path":"conll.py","file_url":"https://github.com/amore-upf/masked-coreference/blob/HEAD/conll.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ea92f536bea4cbd4"}},{"code_sha256_prefix":"5dc251a1bb16c94f","entry":"f1","repo":"amore-upf/masked-coreference","repo_kind":"listed","path":"metrics.py","file_url":"https://github.com/amore-upf/masked-coreference/blob/HEAD/metrics.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5dc251a1bb16c94f"}},{"code_sha256_prefix":"ecab128238ebf253","entry":"gelu","repo":"amore-upf/masked-coreference","repo_kind":"listed","path":"bert/modeling.py","file_url":"https://github.com/amore-upf/masked-coreference/blob/HEAD/bert/modeling.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ecab128238ebf253"}},{"code_sha256_prefix":"50958618b65e514e","entry":"get_assignment_map_from_checkpoint","repo":"amore-upf/masked-coreference","repo_kind":"listed","path":"bert/modeling.py","file_url":"https://github.com/amore-upf/masked-coreference/blob/HEAD/bert/modeling.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"50958618b65e514e"}},{"code_sha256_prefix":"f300a33fe05fc609","entry":"get_doc_key","repo":"amore-upf/masked-coreference","repo_kind":"listed","path":"conll.py","file_url":"https://github.com/amore-upf/masked-coreference/blob/HEAD/conll.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f300a33fe05fc609"}},{"code_sha256_prefix":"de32dc1a0c359c0d","entry":"get_sentence_map","repo":"amore-upf/masked-coreference","repo_kind":"listed","path":"minimize.py","file_url":"https://github.com/amore-upf/masked-coreference/blob/HEAD/minimize.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"de32dc1a0c359c0d"}},{"code_sha256_prefix":"3b625164c1147fcf","entry":"normalize_word","repo":"amore-upf/masked-coreference","repo_kind":"listed","path":"minimize.py","file_url":"https://github.com/amore-upf/masked-coreference/blob/HEAD/minimize.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3b625164c1147fcf"}},{"code_sha256_prefix":"5712dc7b00cdd758","entry":"official_conll_eval","repo":"amore-upf/masked-coreference","repo_kind":"listed","path":"conll.py","file_url":"https://github.com/amore-upf/masked-coreference/blob/HEAD/conll.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5712dc7b00cdd758"}},{"code_sha256_prefix":"edcb69a1bdf0f631","entry":"read_doc_keys","repo":"amore-upf/masked-coreference","repo_kind":"listed","path":"evaluate.py","file_url":"https://github.com/amore-upf/masked-coreference/blob/HEAD/evaluate.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"edcb69a1bdf0f631"}},{"code_sha256_prefix":"3c241ecfe3749a6d","entry":"simple_accuracy","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"3c241ecfe3749a6d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}