{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/characterbert-reconciling-elmo-and-bert-for","title":"CharacterBERT: Reconciling ELMo and BERT for Word-Level Open-Vocabulary Representations From Characters","arxiv_id":"2010.10392","date":"2020-10-20","proceeding":"COLING 2020 8","authors":["Hicham El Boukkouri","Olivier Ferret","Thomas Lavergne","Hiroshi Noji","Pierre Zweigenbaum","Junichi Tsujii"],"abstract":"Due to the compelling improvements brought by BERT, many recent representation models adopted the Transformer architecture as their main building block, consequently inheriting the wordpiece tokenization system despite it not being intrinsically linked to the notion of Transformers. While this system is thought to achieve a good balance between the flexibility of characters and the efficiency of full words, using predefined wordpiece vocabularies from the general domain is not always suitable, especially when building models for specialized domains (e.g., the medical domain). Moreover, adopting a wordpiece tokenization shifts the focus from the word level to the subword level, making the models conceptually more complex and arguably less convenient in practice. For these reasons, we propose CharacterBERT, a new variant of BERT that drops the wordpiece system altogether and uses a Character-CNN module instead to represent entire words by consulting their characters. We show that this new model improves the performance of BERT on a variety of medical domain tasks while at the same time producing robust, word-level and open-vocabulary representations.","url_abs":"https://arxiv.org/abs/2010.10392v3","url_pdf":"https://arxiv.org/pdf/2010.10392v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"characterbert-reconciling-elmo-and-bert-for","repo_url":"https://github.com/helboukkouri/character-bert","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"characterbert-reconciling-elmo-and-bert-for","repo_url":"https://github.com/IMPLabUniPr/UniParma-at-semeval-2021-task-5","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"clinical-concept-extraction","task_name":"Clinical Concept Extraction"},{"task_slug":"drug-drug-interaction-extraction","task_name":"Drug–drug Interaction Extraction"},{"task_slug":"natural-language-inference","task_name":"Natural Language Inference"},{"task_slug":"relation-extraction","task_name":"Relation Extraction"},{"task_slug":"semantic-similarity","task_name":"Semantic Similarity"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"characterbert","method_name":"CharacterBERT"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/clinical-concept-extraction-on-2010-i2b2va","task":"Clinical Concept Extraction","dataset":"2010 i2b2/VA","model":"CharacterBERT (base, medical)","rank_in_archive_order":2,"of":5,"metrics":{"Exact Span F1":"89.24"},"uses_additional_data":false},{"leaderboard":"/sota/drug-drug-interaction-extraction-on-ddi","task":"Drug–drug Interaction Extraction","dataset":"DDI extraction 2013 corpus","model":"CharacterBERT (base, medical)","rank_in_archive_order":9,"of":10,"metrics":{"Micro F1":"80.38"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-mednli","task":"Natural Language Inference","dataset":"MedNLI","model":"CharacterBERT (base, medical)","rank_in_archive_order":4,"of":7,"metrics":{"Accuracy":"84.95"},"uses_additional_data":false},{"leaderboard":"/sota/relation-extraction-on-chemprot","task":"Relation Extraction","dataset":"ChemProt","model":"CharacterBERT (base, medical)","rank_in_archive_order":13,"of":13,"metrics":{"Micro F1":"73.44"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-similarity-on-clinicalsts","task":"Semantic Similarity","dataset":"ClinicalSTS","model":"CharacterBERT (base, medical, ensemble)","rank_in_archive_order":1,"of":1,"metrics":{"Pearson Correlation":"85.62"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2010.10392","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.10392"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/helboukkouri/character-bert","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/IMPLabUniPr/UniParma-at-semeval-2021-task-5","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_honours":2,"ran_draft_wrong":2,"ran":1,"unverified":3},"by_repo_kind":{"listed":{"samples":8,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"211753ba29188d4e","entry":"gelu","repo":"IMPLabUniPr/UniParma-at-semeval-2021-task-5","repo_kind":"listed","path":"transformers/modeling_bert.py","file_url":"https://github.com/IMPLabUniPr/UniParma-at-semeval-2021-task-5/blob/HEAD/transformers/modeling_bert.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"211753ba29188d4e"}},{"code_sha256_prefix":"77601724cad03f95","entry":"gelu_new","repo":"IMPLabUniPr/UniParma-at-semeval-2021-task-5","repo_kind":"listed","path":"transformers/modeling_bert.py","file_url":"https://github.com/IMPLabUniPr/UniParma-at-semeval-2021-task-5/blob/HEAD/transformers/modeling_bert.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"77601724cad03f95"}},{"code_sha256_prefix":"0c145728a35789fa","entry":"load_tf_weights_in_bert","repo":"IMPLabUniPr/UniParma-at-semeval-2021-task-5","repo_kind":"listed","path":"transformers/modeling_bert.py","file_url":"https://github.com/IMPLabUniPr/UniParma-at-semeval-2021-task-5/blob/HEAD/transformers/modeling_bert.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0c145728a35789fa"}},{"code_sha256_prefix":"5abcd8c6705f2368","entry":"squash","repo":"IMPLabUniPr/UniParma-at-semeval-2021-task-5","repo_kind":"listed","path":"modeling/layer_ae.py","file_url":"https://github.com/IMPLabUniPr/UniParma-at-semeval-2021-task-5/blob/HEAD/modeling/layer_ae.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5abcd8c6705f2368"}},{"code_sha256_prefix":"a878dc3112f9ce9e","entry":"tile","repo":"IMPLabUniPr/UniParma-at-semeval-2021-task-5","repo_kind":"listed","path":"transformers/modeling_beam_search.py","file_url":"https://github.com/IMPLabUniPr/UniParma-at-semeval-2021-task-5/blob/HEAD/transformers/modeling_beam_search.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a878dc3112f9ce9e"}},{"code_sha256_prefix":"135b3dc835ffe6ad","entry":"get_confirm_token","repo":"IMPLabUniPr/UniParma-at-semeval-2021-task-5","repo_kind":"listed","path":"download.py","file_url":"https://github.com/IMPLabUniPr/UniParma-at-semeval-2021-task-5/blob/HEAD/download.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"135b3dc835ffe6ad"}},{"code_sha256_prefix":"9ab72e873c485839","entry":"load_classification_dataset","repo":"IMPLabUniPr/UniParma-at-semeval-2021-task-5","repo_kind":"listed","path":"data.py","file_url":"https://github.com/IMPLabUniPr/UniParma-at-semeval-2021-task-5/blob/HEAD/data.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9ab72e873c485839"}},{"code_sha256_prefix":"d4ec0b3bf4bf193d","entry":"squash_v1","repo":"IMPLabUniPr/UniParma-at-semeval-2021-task-5","repo_kind":"listed","path":"modeling/layer_ae.py","file_url":"https://github.com/IMPLabUniPr/UniParma-at-semeval-2021-task-5/blob/HEAD/modeling/layer_ae.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d4ec0b3bf4bf193d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}