{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/wise-a-world-knowledge-informed-semantic","title":"WISE: A World Knowledge-Informed Semantic Evaluation for Text-to-Image Generation","arxiv_id":"2503.07265","date":"2025-03-10","proceeding":null,"authors":["Yuwei Niu","Munan Ning","Mengren Zheng","Bin Lin","Peng Jin","Jiaqi Liao","KunPeng Ning","Bin Zhu","Li Yuan"],"abstract":"Text-to-Image (T2I) models are capable of generating high-quality artistic creations and visual content. However, existing research and evaluation standards predominantly focus on image realism and shallow text-image alignment, lacking a comprehensive assessment of complex semantic understanding and world knowledge integration in text to image generation. To address this challenge, we propose $\\textbf{WISE}$, the first benchmark specifically designed for $\\textbf{W}$orld Knowledge-$\\textbf{I}$nformed $\\textbf{S}$emantic $\\textbf{E}$valuation. WISE moves beyond simple word-pixel mapping by challenging models with 1000 meticulously crafted prompts across 25 sub-domains in cultural common sense, spatio-temporal reasoning, and natural science. To overcome the limitations of traditional CLIP metric, we introduce $\\textbf{WiScore}$, a novel quantitative metric for assessing knowledge-image alignment. Through comprehensive testing of 20 models (10 dedicated T2I models and 10 unified multimodal models) using 1,000 structured prompts spanning 25 subdomains, our findings reveal significant limitations in their ability to effectively integrate and apply world knowledge during image generation, highlighting critical pathways for enhancing knowledge incorporation and application in next-generation T2I models. Code and data are available at https://github.com/PKU-YuanGroup/WISE.","url_abs":"https://arxiv.org/abs/2503.07265v1","url_pdf":"https://arxiv.org/pdf/2503.07265v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"wise-a-world-knowledge-informed-semantic","repo_url":"https://github.com/pku-yuangroup/wise","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"wise-a-world-knowledge-informed-semantic","repo_url":"https://github.com/PKU-YuanGroup/UniWorld-V1","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"common-sense-reasoning","task_name":"Common Sense Reasoning"},{"task_slug":"image-generation","task_name":"Image Generation"},{"task_slug":"text-to-image-generation-1","task_name":"Text to Image Generation"},{"task_slug":"text-to-image-generation","task_name":"Text-to-Image Generation"},{"task_slug":"world-knowledge","task_name":"World Knowledge"}],"methods":[{"method_slug":"clip","method_name":"CLIP"},{"method_slug":"focus","method_name":"Focus"}],"datasets_introduced":[{"slug":"wise","name":"WISE","full_name":""}],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2503.07265","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2503.07265"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/PKU-YuanGroup/WISE","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/PKU-YuanGroup/UniWorld-V1","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/pku-yuangroup/wise","reach":{"status":"ok"}}],"summary":{"ran_honours":1,"ran_draft_wrong":4},"by_repo_kind":{"official":{"samples":5,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":5,"samples":[{"code_sha256_prefix":"9ac25d2c009931dc","entry":"calculate_wiscore","repo":"PKU-YuanGroup/WISE","repo_kind":"official","path":"calculate_verified.py","file_url":"https://github.com/PKU-YuanGroup/WISE/blob/HEAD/calculate_verified.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9ac25d2c009931dc"}},{"code_sha256_prefix":"18429db0bfd705f3","entry":"get_config","repo":"PKU-YuanGroup/WISE","repo_kind":"official","path":"vllm_eval.py","file_url":"https://github.com/PKU-YuanGroup/WISE/blob/HEAD/vllm_eval.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"18429db0bfd705f3"}},{"code_sha256_prefix":"471db2e0891324a8","entry":"load_json","repo":"PKU-YuanGroup/WISE","repo_kind":"official","path":"vllm_eval.py","file_url":"https://github.com/PKU-YuanGroup/WISE/blob/HEAD/vllm_eval.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"471db2e0891324a8"}},{"code_sha256_prefix":"0bc2febf8ef317cf","entry":"load_jsonl","repo":"PKU-YuanGroup/WISE","repo_kind":"official","path":"vllm_eval.py","file_url":"https://github.com/PKU-YuanGroup/WISE/blob/HEAD/vllm_eval.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0bc2febf8ef317cf"}},{"code_sha256_prefix":"09e6d2aa05765680","entry":"process_jsonl_file_segment","repo":"PKU-YuanGroup/WISE","repo_kind":"official","path":"calculate_verified.py","file_url":"https://github.com/PKU-YuanGroup/WISE/blob/HEAD/calculate_verified.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"09e6d2aa05765680"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}