{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/chemberta-large-scale-self-supervised","title":"ChemBERTa: Large-Scale Self-Supervised Pretraining for Molecular Property Prediction","arxiv_id":"2010.09885","date":"2020-10-19","proceeding":null,"authors":["Seyone Chithrananda","Gabriel Grand","Bharath Ramsundar"],"abstract":"GNNs and chemical fingerprints are the predominant approaches to representing molecules for property prediction. However, in NLP, transformers have become the de-facto standard for representation learning thanks to their strong downstream task transfer. In parallel, the software ecosystem around transformers is maturing rapidly, with libraries like HuggingFace and BertViz enabling streamlined training and introspection. In this work, we make one of the first attempts to systematically evaluate transformers on molecular property prediction tasks via our ChemBERTa model. ChemBERTa scales well with pretraining dataset size, offering competitive downstream performance on MoleculeNet and useful attention-based visualization modalities. Our results suggest that transformers offer a promising avenue of future work for molecular representation learning and property prediction. To facilitate these efforts, we release a curated dataset of 77M SMILES from PubChem suitable for large-scale self-supervised pretraining.","url_abs":"https://arxiv.org/abs/2010.09885v2","url_pdf":"https://arxiv.org/pdf/2010.09885v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"chemberta-large-scale-self-supervised","repo_url":"https://github.com/seyonechithrananda/bert-loves-chemistry","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"chemberta-large-scale-self-supervised","repo_url":"https://github.com/SylwiaNowakowska/LLM_Fine_Tuning_Molecular_Properties","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"chemberta-large-scale-self-supervised","repo_url":"https://github.com/XuHangkun/Compound-Protein-Interaction","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"molecular-property-prediction","task_name":"Molecular Property Prediction"},{"task_slug":"prediction","task_name":"Prediction"},{"task_slug":"property-prediction","task_name":"Property Prediction"},{"task_slug":"representation-learning","task_name":"Representation Learning"},{"task_slug":"molecular-representation","task_name":"molecular representation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2010.09885","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.09885"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/XuHangkun/Compound-Protein-Interaction","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/SylwiaNowakowska/LLM_Fine_Tuning_Molecular_Properties","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/seyonechithrananda/bert-loves-chemistry","reach":null}],"summary":{"ran":2,"unverified":1},"by_repo_kind":{"official":{"samples":3,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"9626714e3f9b2b0f","entry":"RobertaClassificationHead","repo":"seyonechithrananda/bert-loves-chemistry","repo_kind":"official","path":"chemberta/utils/roberta_regression.py","file_url":"https://github.com/seyonechithrananda/bert-loves-chemistry/blob/HEAD/chemberta/utils/roberta_regression.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9626714e3f9b2b0f"}},{"code_sha256_prefix":"e831d63c8855521b","entry":"SequenceClassifierOutput","repo":"seyonechithrananda/bert-loves-chemistry","repo_kind":"official","path":"chemberta/utils/roberta_regression.py","file_url":"https://github.com/seyonechithrananda/bert-loves-chemistry/blob/HEAD/chemberta/utils/roberta_regression.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e831d63c8855521b"}},{"code_sha256_prefix":"eaf9bb65f142d995","entry":"RobertaForSequenceClassification","repo":"seyonechithrananda/bert-loves-chemistry","repo_kind":"official","path":"chemberta/utils/roberta_regression.py","file_url":"https://github.com/seyonechithrananda/bert-loves-chemistry/blob/HEAD/chemberta/utils/roberta_regression.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"eaf9bb65f142d995"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}