{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/instructmol-multi-modal-integration-for","title":"InstructMol: Multi-Modal Integration for Building a Versatile and Reliable Molecular Assistant in Drug Discovery","arxiv_id":"2311.16208","date":"2023-11-27","proceeding":null,"authors":["He Cao","Zijing Liu","Xingyu Lu","Yuan YAO","Yu Li"],"abstract":"The rapid evolution of artificial intelligence in drug discovery encounters challenges with generalization and extensive training, yet Large Language Models (LLMs) offer promise in reshaping interactions with complex molecular data. Our novel contribution, InstructMol, a multi-modal LLM, effectively aligns molecular structures with natural language via an instruction-tuning approach, utilizing a two-stage training strategy that adeptly combines limited domain-specific data with molecular and textual information. InstructMol showcases substantial performance improvements in drug discovery-related molecular tasks, surpassing leading LLMs and significantly reducing the gap with specialized models, thereby establishing a robust foundation for a versatile and dependable drug discovery assistant.","url_abs":"https://arxiv.org/abs/2311.16208v2","url_pdf":"https://arxiv.org/pdf/2311.16208v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"instructmol-multi-modal-integration-for","repo_url":"https://github.com/idea-xl/instructmol","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"drug-discovery","task_name":"Drug Discovery"},{"task_slug":"molecule-captioning","task_name":"Molecule Captioning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/molecule-captioning-on-chebi-20","task":"Molecule Captioning","dataset":"ChEBI-20","model":"InstructMol-GS","rank_in_archive_order":32,"of":33,"metrics":{"BLEU-2":"47.5","BLEU-4":"37.1","METEOR":"50.9","ROUGE-1":"56.6","ROUGE-2":"39.4","ROUGE-L":"50.2"},"uses_additional_data":false},{"leaderboard":"/sota/molecule-captioning-on-chebi-20","task":"Molecule Captioning","dataset":"ChEBI-20","model":"InstructMol-G","rank_in_archive_order":33,"of":33,"metrics":{"BLEU-2":"46.6","BLEU-4":"36.5","METEOR":"49.1","ROUGE-1":"54.7","ROUGE-2":"36.5","ROUGE-L":"47.9"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2311.16208","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.16208"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/idea-xl/instructmol","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":3},"by_repo_kind":{"official":{"samples":3,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"c3ee9d07c900dd55","entry":"load_image_from_base64","repo":"idea-xl/instructmol","repo_kind":"official","path":"llava/mm_utils.py","file_url":"https://github.com/idea-xl/instructmol/blob/HEAD/llava/mm_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c3ee9d07c900dd55"}},{"code_sha256_prefix":"1df990c375318896","entry":"process_images","repo":"idea-xl/instructmol","repo_kind":"official","path":"llava/mm_utils.py","file_url":"https://github.com/idea-xl/instructmol/blob/HEAD/llava/mm_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1df990c375318896"}},{"code_sha256_prefix":"7c27eacc7ec807d1","entry":"update_pretrained_config","repo":"idea-xl/instructmol","repo_kind":"official","path":"llava/model/builder.py","file_url":"https://github.com/idea-xl/instructmol/blob/HEAD/llava/model/builder.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7c27eacc7ec807d1"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}