{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/comat-aligning-text-to-image-diffusion-model","title":"CoMat: Aligning Text-to-Image Diffusion Model with Image-to-Text Concept Matching","arxiv_id":"2404.03653","date":"2024-04-04","proceeding":null,"authors":["Dongzhi Jiang","Guanglu Song","Xiaoshi Wu","Renrui Zhang","Dazhong Shen","Zhuofan Zong","Yu Liu","Hongsheng Li"],"abstract":"Diffusion models have demonstrated great success in the field of text-to-image generation. However, alleviating the misalignment between the text prompts and images is still challenging. The root reason behind the misalignment has not been extensively investigated. We observe that the misalignment is caused by inadequate token attention activation. We further attribute this phenomenon to the diffusion model's insufficient condition utilization, which is caused by its training paradigm. To address the issue, we propose CoMat, an end-to-end diffusion model fine-tuning strategy with an image-to-text concept matching mechanism. We leverage an image captioning model to measure image-to-text alignment and guide the diffusion model to revisit ignored tokens. A novel attribute concentration module is also proposed to address the attribute binding problem. Without any image or human preference data, we use only 20K text prompts to fine-tune SDXL to obtain CoMat-SDXL. Extensive experiments show that CoMat-SDXL significantly outperforms the baseline model SDXL in two text-to-image alignment benchmarks and achieves start-of-the-art performance.","url_abs":"https://arxiv.org/abs/2404.03653v3","url_pdf":"https://arxiv.org/pdf/2404.03653v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"comat-aligning-text-to-image-diffusion-model","repo_url":"https://github.com/caraj7/comat","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"comat-aligning-text-to-image-diffusion-model","repo_url":"https://github.com/mlpc-ucsd/TokenCompose","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"attribute","task_name":"Attribute"},{"task_slug":"image-captioning","task_name":"Image Captioning"},{"task_slug":"image-generation","task_name":"Image Generation"},{"task_slug":"image-to-text","task_name":"Image to text"},{"task_slug":"text-to-image-generation-1","task_name":"Text to Image Generation"},{"task_slug":"text-to-image-generation","task_name":"Text-to-Image Generation"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2404.03653","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.03653"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mlpc-ucsd/TokenCompose","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/caraj7/comat","reach":{"status":"ok"}}],"summary":{"ran":5,"unverified":4},"by_repo_kind":{"official":{"samples":9,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":9,"samples":[{"code_sha256_prefix":"41b85d0d5fe472eb","entry":"align_wordpieces_indices","repo":"caraj7/comat","repo_kind":"official","path":"attribute_concen_utils.py","file_url":"https://github.com/caraj7/comat/blob/HEAD/attribute_concen_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"41b85d0d5fe472eb"}},{"code_sha256_prefix":"52a421d8ab2f0163","entry":"get_cross_attn_map_from_unet","repo":"caraj7/comat","repo_kind":"official","path":"attn_utils/tc_attn_utils.py","file_url":"https://github.com/caraj7/comat/blob/HEAD/attn_utils/tc_attn_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"52a421d8ab2f0163"}},{"code_sha256_prefix":"b2840876374a5c28","entry":"get_word_idx","repo":"caraj7/comat","repo_kind":"official","path":"attn_utils/tc_loss_utils.py","file_url":"https://github.com/caraj7/comat/blob/HEAD/attn_utils/tc_loss_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b2840876374a5c28"}},{"code_sha256_prefix":"0f0e69c734e68a77","entry":"is_sublist","repo":"caraj7/comat","repo_kind":"official","path":"AttrConcenTrainableSDPipeline.py","file_url":"https://github.com/caraj7/comat/blob/HEAD/AttrConcenTrainableSDPipeline.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0f0e69c734e68a77"}},{"code_sha256_prefix":"5b4d92944d2eae76","entry":"unify_lists","repo":"caraj7/comat","repo_kind":"official","path":"AttrConcenTrainableSDPipeline.py","file_url":"https://github.com/caraj7/comat/blob/HEAD/AttrConcenTrainableSDPipeline.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5b4d92944d2eae76"}},{"code_sha256_prefix":"9d20fb1f7ebc2ef7","entry":"extract_attribution_indices","repo":"caraj7/comat","repo_kind":"official","path":"attribute_concen_utils.py","file_url":"https://github.com/caraj7/comat/blob/HEAD/attribute_concen_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9d20fb1f7ebc2ef7"}},{"code_sha256_prefix":"67753c04344576ce","entry":"extract_attribution_indices_with_verbs","repo":"caraj7/comat","repo_kind":"official","path":"attribute_concen_utils.py","file_url":"https://github.com/caraj7/comat/blob/HEAD/attribute_concen_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"67753c04344576ce"}},{"code_sha256_prefix":"ec4bc8b3b8826e0b","entry":"get_grounding_loss_by_layer","repo":"caraj7/comat","repo_kind":"official","path":"attn_utils/tc_loss_utils.py","file_url":"https://github.com/caraj7/comat/blob/HEAD/attn_utils/tc_loss_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ec4bc8b3b8826e0b"}},{"code_sha256_prefix":"e4da54ebde4a3e29","entry":"register_attention_control","repo":"caraj7/comat","repo_kind":"official","path":"attn_utils/tc_attn_utils.py","file_url":"https://github.com/caraj7/comat/blob/HEAD/attn_utils/tc_attn_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e4da54ebde4a3e29"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}