{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/exploring-the-deep-fusion-of-large-language","title":"Exploring the Deep Fusion of Large Language Models and Diffusion Transformers for Text-to-Image Synthesis","arxiv_id":"2505.10046","date":"2025-05-15","proceeding":"CVPR 2025 1","authors":["Bingda Tang","BoYang Zheng","Xichen Pan","Sayak Paul","Saining Xie"],"abstract":"This paper does not describe a new method; instead, it provides a thorough exploration of an important yet understudied design space related to recent advances in text-to-image synthesis -- specifically, the deep fusion of large language models (LLMs) and diffusion transformers (DiTs) for multi-modal generation. Previous studies mainly focused on overall system performance rather than detailed comparisons with alternative methods, and key design details and training recipes were often left undisclosed. These gaps create uncertainty about the real potential of this approach. To fill these gaps, we conduct an empirical study on text-to-image generation, performing controlled comparisons with established baselines, analyzing important design choices, and providing a clear, reproducible recipe for training at scale. We hope this work offers meaningful data points and practical guidelines for future research in multi-modal generation.","url_abs":"https://arxiv.org/abs/2505.10046v1","url_pdf":"https://arxiv.org/pdf/2505.10046v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"exploring-the-deep-fusion-of-large-language","repo_url":"https://github.com/tang-bd/fuse-dit","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"image-generation","task_name":"Image Generation"},{"task_slug":"text-to-image-generation-1","task_name":"Text to Image Generation"},{"task_slug":"text-to-image-generation","task_name":"Text-to-Image Generation"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2505.10046","atlas_url":"https://app.syntology.ai/?focus=2505.10046","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.10046"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/tang-bd/fuse-dit","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":6,"unverified":3},"by_repo_kind":{"official":{"samples":9,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"b404601d7b9085a4","entry":"llm_collate_fn","repo":"tang-bd/fuse-dit","repo_kind":"official","path":"diffusion/data.py","file_url":"https://github.com/tang-bd/fuse-dit/blob/HEAD/diffusion/data.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b404601d7b9085a4"}},{"code_sha256_prefix":"0d69313c0ca386df","entry":"llm_preprocess_fn","repo":"tang-bd/fuse-dit","repo_kind":"official","path":"diffusion/data.py","file_url":"https://github.com/tang-bd/fuse-dit/blob/HEAD/diffusion/data.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0d69313c0ca386df"}},{"code_sha256_prefix":"cc333e67545ba03b","entry":"rank_zero_only","repo":"tang-bd/fuse-dit","repo_kind":"official","path":"diffusion/trainers.py","file_url":"https://github.com/tang-bd/fuse-dit/blob/HEAD/diffusion/trainers.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cc333e67545ba03b"}},{"code_sha256_prefix":"0582c403b48a1d8d","entry":"retrieve_timesteps","repo":"tang-bd/fuse-dit","repo_kind":"official","path":"diffusion/pipelines.py","file_url":"https://github.com/tang-bd/fuse-dit/blob/HEAD/diffusion/pipelines.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0582c403b48a1d8d"}},{"code_sha256_prefix":"670d270293f0f7af","entry":"update_cross_attention_mask","repo":"tang-bd/fuse-dit","repo_kind":"official","path":"diffusion/models.py","file_url":"https://github.com/tang-bd/fuse-dit/blob/HEAD/diffusion/models.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"670d270293f0f7af"}},{"code_sha256_prefix":"a6c478310fa2b73c","entry":"update_self_attention_mask","repo":"tang-bd/fuse-dit","repo_kind":"official","path":"diffusion/models.py","file_url":"https://github.com/tang-bd/fuse-dit/blob/HEAD/diffusion/models.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a6c478310fa2b73c"}},{"code_sha256_prefix":"c48edbd3d7f74874","entry":"get_llm","repo":"tang-bd/fuse-dit","repo_kind":"official","path":"diffusion/models.py","file_url":"https://github.com/tang-bd/fuse-dit/blob/HEAD/diffusion/models.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c48edbd3d7f74874"}},{"code_sha256_prefix":"466cc1b9bd0f12ca","entry":"sync_to_cpu","repo":"tang-bd/fuse-dit","repo_kind":"official","path":"diffusion/trainers.py","file_url":"https://github.com/tang-bd/fuse-dit/blob/HEAD/diffusion/trainers.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"466cc1b9bd0f12ca"}},{"code_sha256_prefix":"89a0489155e4faeb","entry":"unwrap","repo":"tang-bd/fuse-dit","repo_kind":"official","path":"diffusion/trainers.py","file_url":"https://github.com/tang-bd/fuse-dit/blob/HEAD/diffusion/trainers.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"89a0489155e4faeb"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}