{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/glyphcontrol-glyph-conditional-control-for-1","title":"GlyphControl: Glyph Conditional Control for Visual Text Generation","arxiv_id":"2305.18259","date":"2023-05-29","proceeding":"NeurIPS 2023 11","authors":["Yukang Yang","Dongnan Gui","Yuhui Yuan","Weicong Liang","Haisong Ding","Han Hu","Kai Chen"],"abstract":"Recently, there has been an increasing interest in developing diffusion-based text-to-image generative models capable of generating coherent and well-formed visual text. In this paper, we propose a novel and efficient approach called GlyphControl to address this task. Unlike existing methods that rely on character-aware text encoders like ByT5 and require retraining of text-to-image models, our approach leverages additional glyph conditional information to enhance the performance of the off-the-shelf Stable-Diffusion model in generating accurate visual text. By incorporating glyph instructions, users can customize the content, location, and size of the generated text according to their specific requirements. To facilitate further research in visual text generation, we construct a training benchmark dataset called LAION-Glyph. We evaluate the effectiveness of our approach by measuring OCR-based metrics, CLIP score, and FID of the generated visual text. Our empirical evaluations demonstrate that GlyphControl outperforms the recent DeepFloyd IF approach in terms of OCR accuracy, CLIP score, and FID, highlighting the efficacy of our method.","url_abs":"https://arxiv.org/abs/2305.18259v2","url_pdf":"https://arxiv.org/pdf/2305.18259v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"glyphcontrol-glyph-conditional-control-for-1","repo_url":"https://github.com/aigtext/glyphcontrol-release","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"optical-character-recognition","task_name":"Optical Character Recognition (OCR)"},{"task_slug":"text-generation","task_name":"Text Generation"}],"methods":[{"method_slug":"clip","method_name":"CLIP"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2305.18259","atlas_url":"https://app.syntology.ai/?focus=2305.18259","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.18259"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/aigtext/glyphcontrol-release","reach":null}],"summary":{"ran":4,"unverified":5},"by_repo_kind":{"official":{"samples":9,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"8f64e619c23619c5","entry":"CrossAttention","repo":"aigtext/glyphcontrol-release","repo_kind":"official","path":"cldm/cldm.py","file_url":"https://github.com/aigtext/glyphcontrol-release/blob/HEAD/cldm/cldm.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8f64e619c23619c5"}},{"code_sha256_prefix":"2c8910dba7592809","entry":"MemoryEfficientCrossAttention","repo":"aigtext/glyphcontrol-release","repo_kind":"official","path":"cldm/cldm.py","file_url":"https://github.com/aigtext/glyphcontrol-release/blob/HEAD/cldm/cldm.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2c8910dba7592809"}},{"code_sha256_prefix":"4bed2e5daf64df72","entry":"QKVAttention","repo":"aigtext/glyphcontrol-release","repo_kind":"official","path":"cldm/cldm.py","file_url":"https://github.com/aigtext/glyphcontrol-release/blob/HEAD/cldm/cldm.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4bed2e5daf64df72"}},{"code_sha256_prefix":"ba0567fc5b71dbf6","entry":"QKVAttentionLegacy","repo":"aigtext/glyphcontrol-release","repo_kind":"official","path":"cldm/cldm.py","file_url":"https://github.com/aigtext/glyphcontrol-release/blob/HEAD/cldm/cldm.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ba0567fc5b71dbf6"}},{"code_sha256_prefix":"e59c125335cdce0a","entry":"AttentionBlock","repo":"aigtext/glyphcontrol-release","repo_kind":"official","path":"cldm/cldm.py","file_url":"https://github.com/aigtext/glyphcontrol-release/blob/HEAD/cldm/cldm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e59c125335cdce0a"}},{"code_sha256_prefix":"c8644b25d74977f2","entry":"BasicTransformerBlock","repo":"aigtext/glyphcontrol-release","repo_kind":"official","path":"cldm/cldm.py","file_url":"https://github.com/aigtext/glyphcontrol-release/blob/HEAD/cldm/cldm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c8644b25d74977f2"}},{"code_sha256_prefix":"1e1a7f7b69eb27b1","entry":"ControlNet","repo":"aigtext/glyphcontrol-release","repo_kind":"official","path":"cldm/cldm.py","file_url":"https://github.com/aigtext/glyphcontrol-release/blob/HEAD/cldm/cldm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1e1a7f7b69eb27b1"}},{"code_sha256_prefix":"97e5c1d4532076bf","entry":"SpatialTransformer","repo":"aigtext/glyphcontrol-release","repo_kind":"official","path":"cldm/cldm.py","file_url":"https://github.com/aigtext/glyphcontrol-release/blob/HEAD/cldm/cldm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"97e5c1d4532076bf"}},{"code_sha256_prefix":"48f3e455f9262eb2","entry":"TimestepEmbedSequential","repo":"aigtext/glyphcontrol-release","repo_kind":"official","path":"cldm/cldm.py","file_url":"https://github.com/aigtext/glyphcontrol-release/blob/HEAD/cldm/cldm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"48f3e455f9262eb2"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}