{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/multimodal-large-language-models-for-inverse","title":"Multimodal Large Language Models for Inverse Molecular Design with Retrosynthetic Planning","arxiv_id":"2410.04223","date":"2024-10-05","proceeding":null,"authors":["Gang Liu","Michael Sun","Wojciech Matusik","Meng Jiang","Jie Chen"],"abstract":"While large language models (LLMs) have integrated images, adapting them to graphs remains challenging, limiting their applications in materials and drug design. This difficulty stems from the need for coherent autoregressive generation across texts and graphs. To address this, we introduce Llamole, the first multimodal LLM capable of interleaved text and graph generation, enabling molecular inverse design with retrosynthetic planning. Llamole integrates a base LLM with the Graph Diffusion Transformer and Graph Neural Networks for multi-conditional molecular generation and reaction inference within texts, while the LLM, with enhanced molecular understanding, flexibly controls activation among the different graph modules. Additionally, Llamole integrates A* search with LLM-based cost functions for efficient retrosynthetic planning. We create benchmarking datasets and conduct extensive experiments to evaluate Llamole against in-context learning and supervised fine-tuning. Llamole significantly outperforms 14 adapted LLMs across 12 metrics for controllable molecular design and retrosynthetic planning.","url_abs":"https://arxiv.org/abs/2410.04223v1","url_pdf":"https://arxiv.org/pdf/2410.04223v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"multimodal-large-language-models-for-inverse","repo_url":"https://github.com/liugangcode/Llamole","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"benchmarking","task_name":"Benchmarking"},{"task_slug":"drug-design","task_name":"Drug Design"},{"task_slug":"graph-generation","task_name":"Graph Generation"},{"task_slug":"in-context-learning","task_name":"In-Context Learning"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"base","method_name":"BASE"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"diffusion","method_name":"Diffusion"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[{"slug":"molqa","name":"MolQA","full_name":""}],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2410.04223","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.04223"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/liugangcode/Llamole","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":9,"unverified":2},"by_repo_kind":{"official":{"samples":11,"ran":9,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"684cc182c9aa3bac","entry":"convert_to_dict","repo":"liugangcode/Llamole","repo_kind":"official","path":"src/model/modeling_llamole.py","file_url":"https://github.com/liugangcode/Llamole/blob/HEAD/src/model/modeling_llamole.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"684cc182c9aa3bac"}},{"code_sha256_prefix":"e493575b4567fc5a","entry":"count_parameters","repo":"liugangcode/Llamole","repo_kind":"official","path":"src/extras/misc.py","file_url":"https://github.com/liugangcode/Llamole/blob/HEAD/src/extras/misc.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e493575b4567fc5a"}},{"code_sha256_prefix":"87f7bd0b36b591d2","entry":"dict_to_list","repo":"liugangcode/Llamole","repo_kind":"official","path":"src/eval/dataset.py","file_url":"https://github.com/liugangcode/Llamole/blob/HEAD/src/eval/dataset.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"87f7bd0b36b591d2"}},{"code_sha256_prefix":"3c781f40e814fc9a","entry":"format_example","repo":"liugangcode/Llamole","repo_kind":"official","path":"launch.py","file_url":"https://github.com/liugangcode/Llamole/blob/HEAD/launch.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3c781f40e814fc9a"}},{"code_sha256_prefix":"4049a2f89b11dd2f","entry":"get_logger","repo":"liugangcode/Llamole","repo_kind":"official","path":"src/extras/logging.py","file_url":"https://github.com/liugangcode/Llamole/blob/HEAD/src/extras/logging.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4049a2f89b11dd2f"}},{"code_sha256_prefix":"500cccdb496451fd","entry":"has_tokenized_data","repo":"liugangcode/Llamole","repo_kind":"official","path":"src/extras/misc.py","file_url":"https://github.com/liugangcode/Llamole/blob/HEAD/src/extras/misc.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"500cccdb496451fd"}},{"code_sha256_prefix":"9a4743e426fdc4ab","entry":"process_property","repo":"liugangcode/Llamole","repo_kind":"official","path":"launch.py","file_url":"https://github.com/liugangcode/Llamole/blob/HEAD/launch.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9a4743e426fdc4ab"}},{"code_sha256_prefix":"952ea691225d8618","entry":"remove_extra_spaces","repo":"liugangcode/Llamole","repo_kind":"official","path":"src/eval/workflow.py","file_url":"https://github.com/liugangcode/Llamole/blob/HEAD/src/eval/workflow.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"952ea691225d8618"}},{"code_sha256_prefix":"0b9f48a781766d05","entry":"smooth","repo":"liugangcode/Llamole","repo_kind":"official","path":"src/extras/ploting.py","file_url":"https://github.com/liugangcode/Llamole/blob/HEAD/src/extras/ploting.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0b9f48a781766d05"}},{"code_sha256_prefix":"b0a7fecf74187af6","entry":"gen_loss_plot","repo":"liugangcode/Llamole","repo_kind":"official","path":"src/extras/ploting.py","file_url":"https://github.com/liugangcode/Llamole/blob/HEAD/src/extras/ploting.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b0a7fecf74187af6"}},{"code_sha256_prefix":"af446e6ee5be84b9","entry":"infer_optim_dtype","repo":"liugangcode/Llamole","repo_kind":"official","path":"src/extras/misc.py","file_url":"https://github.com/liugangcode/Llamole/blob/HEAD/src/extras/misc.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"af446e6ee5be84b9"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}