{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/git-mol-a-multi-modal-large-language-model","title":"GIT-Mol: A Multi-modal Large Language Model for Molecular Science with Graph, Image, and Text","arxiv_id":"2308.06911","date":"2023-08-14","proceeding":null,"authors":["PengFei Liu","Yiming Ren","Jun Tao","Zhixiang Ren"],"abstract":"Large language models have made significant strides in natural language processing, enabling innovative applications in molecular science by processing textual representations of molecules. However, most existing language models cannot capture the rich information with complex molecular structures or images. In this paper, we introduce GIT-Mol, a multi-modal large language model that integrates the Graph, Image, and Text information. To facilitate the integration of multi-modal molecular data, we propose GIT-Former, a novel architecture that is capable of aligning all modalities into a unified latent space. We achieve a 5%-10% accuracy increase in properties prediction and a 20.2% boost in molecule generation validity compared to the baselines. With the any-to-language molecular translation strategy, our model has the potential to perform more downstream tasks, such as compound name recognition and chemical reaction prediction.","url_abs":"https://arxiv.org/abs/2308.06911v3","url_pdf":"https://arxiv.org/pdf/2308.06911v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"git-mol-a-multi-modal-large-language-model","repo_url":"https://github.com/ai-hpc-research-team/git-mol","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"drug-discovery","task_name":"Drug Discovery"},{"task_slug":"image-captioning","task_name":"Image Captioning"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"large-language-model","task_name":"Large Language Model"},{"task_slug":"molecule-captioning","task_name":"Molecule Captioning"},{"task_slug":"property-prediction","task_name":"Property Prediction"},{"task_slug":"text-based-de-novo-molecule-generation","task_name":"Text-based de novo Molecule Generation"},{"task_slug":"molecular-representation","task_name":"molecular representation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/drug-discovery-on-bace","task":"Drug Discovery","dataset":"BACE","model":"GIT-Mol(G+S)","rank_in_archive_order":6,"of":6,"metrics":{"AUC":"0.8108"},"uses_additional_data":false},{"leaderboard":"/sota/drug-discovery-on-bbbp","task":"Drug Discovery","dataset":"BBBP","model":"GIT-Mol(G+S)","rank_in_archive_order":3,"of":4,"metrics":{"AUC":"0.739"},"uses_additional_data":false},{"leaderboard":"/sota/drug-discovery-on-sider","task":"Drug Discovery","dataset":"SIDER","model":"GIT-Mol(G+S)","rank_in_archive_order":3,"of":4,"metrics":{"AUC":"0.634"},"uses_additional_data":false},{"leaderboard":"/sota/drug-discovery-on-tox21","task":"Drug Discovery","dataset":"Tox21","model":"GIT-Mol(G+S)","rank_in_archive_order":10,"of":11,"metrics":{"AUC":"0.759"},"uses_additional_data":false},{"leaderboard":"/sota/drug-discovery-on-toxcast","task":"Drug Discovery","dataset":"ToxCast","model":"GIT-Mol(G+S)","rank_in_archive_order":4,"of":5,"metrics":{"AUC":"0.668"},"uses_additional_data":false},{"leaderboard":"/sota/drug-discovery-on-clintox","task":"Drug Discovery","dataset":"clintox","model":"GIT-Mol(G+S)","rank_in_archive_order":3,"of":4,"metrics":{"AUC":"0.883"},"uses_additional_data":false},{"leaderboard":"/sota/image-captioning-on-chebi-20","task":"Image Captioning","dataset":"ChEBI-20","model":"GIT-Mol","rank_in_archive_order":1,"of":1,"metrics":{"BLEU":"0.924","Exact":"0.461","Levenshtein":"6.575","MACCS FTS":"0.962","Morgan FTS":"0.894","RDK FTS":"0.906","Validity":"0.899"},"uses_additional_data":false},{"leaderboard":"/sota/text-based-de-novo-molecule-generation-on","task":"Text-based de novo Molecule Generation","dataset":"ChEBI-20","model":"GIT-Mol-caption","rank_in_archive_order":16,"of":20,"metrics":{"BLEU":"75.6","Exact Match":"5.1","Levenshtein":"26.315","MACCS FTS":"73.8","Morgan FTS":"51.9","RDK FTS":"58.2","Validity":"92.8"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2308.06911","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2308.06911"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ai-hpc-research-team/git-mol","reach":null}],"summary":{"ran_draft_wrong":1},"by_repo_kind":{"official":{"samples":1,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"138b87fa967c256f","entry":"get_latest_checkpoint","repo":"ai-hpc-research-team/git-mol","repo_kind":"official","path":"train/pretrain.py","file_url":"https://github.com/ai-hpc-research-team/git-mol/blob/HEAD/train/pretrain.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"138b87fa967c256f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}