{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/scifive-a-text-to-text-transformer-model-for","title":"SciFive: a text-to-text transformer model for biomedical literature","arxiv_id":"2106.03598","date":"2021-05-28","proceeding":null,"authors":["Long N. Phan","James T. Anibal","Hieu Tran","Shaurya Chanana","Erol Bahadroglu","Alec Peltekian","Grégoire Altan-Bonnet"],"abstract":"In this report, we introduce SciFive, a domain-specific T5 model that has been pre-trained on large biomedical corpora. Our model outperforms the current SOTA methods (i.e. BERT, BioBERT, Base T5) on tasks in named entity relation, relation extraction, natural language inference, and question-answering. We show that text-generation methods have significant potential in a broad array of biomedical NLP tasks, particularly those requiring longer, more complex outputs. Our results support the exploration of more difficult text generation tasks and the development of new methods in this area","url_abs":"https://arxiv.org/abs/2106.03598v1","url_pdf":"https://arxiv.org/pdf/2106.03598v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"scifive-a-text-to-text-transformer-model-for","repo_url":"https://github.com/justinphan3110/SciFive","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"jax","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"document-classification","task_name":"Document Classification"},{"task_slug":"drug-drug-interaction-extraction","task_name":"Drug–drug Interaction Extraction"},{"task_slug":"named-entity-recognition-ner","task_name":"Named Entity Recognition (NER)"},{"task_slug":"natural-language-inference","task_name":"Natural Language Inference"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":null,"task_name":"Relation"},{"task_slug":"relation-extraction","task_name":"Relation Extraction"},{"task_slug":"text-generation","task_name":"Text Generation"}],"methods":[{"method_slug":"adafactor","method_name":"Adafactor"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"glu","method_name":"Gated Linear Unit"},{"method_slug":"inverse-square-root-schedule","method_name":"Inverse Square Root Schedule"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"sentencepiece","method_name":"SentencePiece"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"t5","method_name":"T5"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/document-classification-on-hoc","task":"Document Classification","dataset":"HOC","model":"SciFive-large","rank_in_archive_order":3,"of":5,"metrics":{"F1":"86.08"},"uses_additional_data":false},{"leaderboard":"/sota/drug-drug-interaction-extraction-on-ddi","task":"Drug–drug Interaction Extraction","dataset":"DDI extraction 2013 corpus","model":"SciFive-large","rank_in_archive_order":2,"of":10,"metrics":{"F1":"0.8367","Micro F1":"83.67"},"uses_additional_data":false},{"leaderboard":"/sota/named-entity-recognition-on-bc5cdr-chemical","task":"Named Entity Recognition (NER)","dataset":"BC5CDR-chemical","model":"SciFive-Large","rank_in_archive_order":2,"of":13,"metrics":{"F1":"94.76"},"uses_additional_data":false},{"leaderboard":"/sota/named-entity-recognition-on-bc5cdr-disease","task":"Named Entity Recognition (NER)","dataset":"BC5CDR-disease","model":"SciFive-Large","rank_in_archive_order":3,"of":10,"metrics":{"F1":"87.62"},"uses_additional_data":false},{"leaderboard":"/sota/named-entity-recognition-ner-on-jnlpba","task":"Named Entity Recognition (NER)","dataset":"JNLPBA","model":"SciFive-Large","rank_in_archive_order":14,"of":17,"metrics":{"F1":"77.55"},"uses_additional_data":false},{"leaderboard":"/sota/named-entity-recognition-ner-on-ncbi-disease","task":"Named Entity Recognition (NER)","dataset":"NCBI-disease","model":"SciFive-Base","rank_in_archive_order":3,"of":26,"metrics":{"F1":"89.39"},"uses_additional_data":false},{"leaderboard":"/sota/named-entity-recognition-on-species-800","task":"Named Entity Recognition (NER)","dataset":"Species-800","model":"SciFive-Base","rank_in_archive_order":3,"of":4,"metrics":{"F1":"76.55"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-mednli","task":"Natural Language Inference","dataset":"MedNLI","model":"SciFive-large","rank_in_archive_order":2,"of":7,"metrics":{"Accuracy":"86.57","Params (M)":"738"},"uses_additional_data":false},{"leaderboard":"/sota/relation-extraction-on-chemprot","task":"Relation Extraction","dataset":"ChemProt","model":"SciFive Large","rank_in_archive_order":4,"of":13,"metrics":{"F1":"78"},"uses_additional_data":false},{"leaderboard":"/sota/relation-extraction-on-chemprot","task":"Relation Extraction","dataset":"ChemProt","model":"BioT5X (base)","rank_in_archive_order":6,"of":13,"metrics":{"F1":"77.40"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2106.03598","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2106.03598"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/justinphan3110/SciFive","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":5},"by_repo_kind":{"official":{"samples":5,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"1be5efa4c923f993","entry":"accuracy","repo":"justinphan3110/SciFive","repo_kind":"official","path":"biot5x/src/metrics.py","file_url":"https://github.com/justinphan3110/SciFive/blob/HEAD/biot5x/src/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1be5efa4c923f993"}},{"code_sha256_prefix":"dac0288079ac279f","entry":"dot_product_attention","repo":"justinphan3110/SciFive","repo_kind":"official","path":"biot5x/configs/t5/layers.py","file_url":"https://github.com/justinphan3110/SciFive/blob/HEAD/biot5x/configs/t5/layers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dac0288079ac279f"}},{"code_sha256_prefix":"12ff36b88687dd69","entry":"f1","repo":"justinphan3110/SciFive","repo_kind":"official","path":"biot5x/src/metrics.py","file_url":"https://github.com/justinphan3110/SciFive/blob/HEAD/biot5x/src/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"12ff36b88687dd69"}},{"code_sha256_prefix":"1184cae8ca10c644","entry":"make_attention_mask","repo":"justinphan3110/SciFive","repo_kind":"official","path":"biot5x/configs/t5/layers.py","file_url":"https://github.com/justinphan3110/SciFive/blob/HEAD/biot5x/configs/t5/layers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1184cae8ca10c644"}},{"code_sha256_prefix":"94c13ffc5f28dfba","entry":"make_causal_mask","repo":"justinphan3110/SciFive","repo_kind":"official","path":"biot5x/configs/t5/layers.py","file_url":"https://github.com/justinphan3110/SciFive/blob/HEAD/biot5x/configs/t5/layers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"94c13ffc5f28dfba"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}