{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/gligen-open-set-grounded-text-to-image","title":"GLIGEN: Open-Set Grounded Text-to-Image Generation","arxiv_id":"2301.07093","date":"2023-01-17","proceeding":"CVPR 2023 1","authors":["Yuheng Li","Haotian Liu","Qingyang Wu","Fangzhou Mu","Jianwei Yang","Jianfeng Gao","Chunyuan Li","Yong Jae Lee"],"abstract":"Large-scale text-to-image diffusion models have made amazing advances. However, the status quo is to use text input alone, which can impede controllability. In this work, we propose GLIGEN, Grounded-Language-to-Image Generation, a novel approach that builds upon and extends the functionality of existing pre-trained text-to-image diffusion models by enabling them to also be conditioned on grounding inputs. To preserve the vast concept knowledge of the pre-trained model, we freeze all of its weights and inject the grounding information into new trainable layers via a gated mechanism. Our model achieves open-world grounded text2img generation with caption and bounding box condition inputs, and the grounding ability generalizes well to novel spatial configurations and concepts. GLIGEN's zero-shot performance on COCO and LVIS outperforms that of existing supervised layout-to-image baselines by a large margin.","url_abs":"https://arxiv.org/abs/2301.07093v2","url_pdf":"https://arxiv.org/pdf/2301.07093v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"gligen-open-set-grounded-text-to-image","repo_url":"https://github.com/gligen/GLIGEN","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"conditional-text-to-image-synthesis","task_name":"Conditional Text-to-Image Synthesis"},{"task_slug":"image-generation","task_name":"Image Generation"},{"task_slug":"image-inpainting","task_name":"Image Inpainting"},{"task_slug":"layout-to-image-generation","task_name":"Layout-to-Image Generation"},{"task_slug":"text-to-image-generation-1","task_name":"Text to Image Generation"},{"task_slug":"text-to-image-generation","task_name":"Text-to-Image Generation"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/conditional-text-to-image-synthesis-on-coco-1","task":"Conditional Text-to-Image Synthesis","dataset":"COCO-MIG","model":"Gligen (zero-shot)","rank_in_archive_order":4,"of":5,"metrics":{"instance success rate":"0.30","mIoU":"0.27"},"uses_additional_data":false},{"leaderboard":"/sota/layout-to-image-generation-on-layoutbench-4","task":"Layout-to-Image Generation","dataset":"LayoutBench-COCO - Combination","model":"GLIGEN","rank_in_archive_order":2,"of":4,"metrics":{"AP":"36.3"},"uses_additional_data":false},{"leaderboard":"/sota/layout-to-image-generation-on-layoutbench-1","task":"Layout-to-Image Generation","dataset":"LayoutBench-COCO - Number","model":"GLIGEN","rank_in_archive_order":3,"of":4,"metrics":{"AP":"30.7"},"uses_additional_data":false},{"leaderboard":"/sota/layout-to-image-generation-on-layoutbench-2","task":"Layout-to-Image Generation","dataset":"LayoutBench-COCO - Position","model":"GLIGEN","rank_in_archive_order":2,"of":4,"metrics":{"AP":"38.9"},"uses_additional_data":false},{"leaderboard":"/sota/layout-to-image-generation-on-layoutbench-3","task":"Layout-to-Image Generation","dataset":"LayoutBench-COCO - Size","model":"GLIGEN","rank_in_archive_order":2,"of":4,"metrics":{"AP":"33.3"},"uses_additional_data":false},{"leaderboard":"/sota/text-to-image-generation-on-coco","task":"Text-to-Image Generation","dataset":"COCO (Common Objects in Context)","model":"GLIGEN (fine-tuned, Detection + Caption data)","rank_in_archive_order":4,"of":69,"metrics":{"FID":"5.61"},"uses_additional_data":false},{"leaderboard":"/sota/text-to-image-generation-on-coco","task":"Text-to-Image Generation","dataset":"COCO (Common Objects in Context)","model":"GLIGEN (fine-tuned, Detection data only)","rank_in_archive_order":5,"of":69,"metrics":{"FID":"5.82"},"uses_additional_data":false},{"leaderboard":"/sota/text-to-image-generation-on-coco","task":"Text-to-Image Generation","dataset":"COCO (Common Objects in Context)","model":"GLIGEN (fine-tuned, Grounding data)","rank_in_archive_order":9,"of":69,"metrics":{"FID":"6.38"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2301.07093","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2301.07093"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/gligen/GLIGEN","reach":null}],"summary":{"ran":2},"by_repo_kind":{"official":{"samples":2,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"507c6b1494a9654d","entry":"FourierEmbedder","repo":"gligen/GLIGEN","repo_kind":"official","path":"ldm/modules/diffusionmodules/text_image_grounding_net.py","file_url":"https://github.com/gligen/GLIGEN/blob/HEAD/ldm/modules/diffusionmodules/text_image_grounding_net.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"507c6b1494a9654d"}},{"code_sha256_prefix":"ebf186a07a3f13ba","entry":"PositionNet","repo":"gligen/GLIGEN","repo_kind":"official","path":"ldm/modules/diffusionmodules/text_image_grounding_net.py","file_url":"https://github.com/gligen/GLIGEN/blob/HEAD/ldm/modules/diffusionmodules/text_image_grounding_net.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ebf186a07a3f13ba"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}