{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/imagereward-learning-and-evaluating-human-1","title":"ImageReward: Learning and Evaluating Human Preferences for Text-to-Image Generation","arxiv_id":"2304.05977","date":"2023-04-12","proceeding":"NeurIPS 2023 11","authors":["Jiazheng Xu","Xiao Liu","Yuchen Wu","Yuxuan Tong","Qinkai Li","Ming Ding","Jie Tang","Yuxiao Dong"],"abstract":"We present a comprehensive solution to learn and improve text-to-image models from human preference feedback. To begin with, we build ImageReward -- the first general-purpose text-to-image human preference reward model -- to effectively encode human preferences. Its training is based on our systematic annotation pipeline including rating and ranking, which collects 137k expert comparisons to date. In human evaluation, ImageReward outperforms existing scoring models and metrics, making it a promising automatic metric for evaluating text-to-image synthesis. On top of it, we propose Reward Feedback Learning (ReFL), a direct tuning algorithm to optimize diffusion models against a scorer. Both automatic and human evaluation support ReFL's advantages over compared methods. All code and datasets are provided at \\url{https://github.com/THUDM/ImageReward}.","url_abs":"https://arxiv.org/abs/2304.05977v4","url_pdf":"https://arxiv.org/pdf/2304.05977v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"imagereward-learning-and-evaluating-human-1","repo_url":"https://github.com/thudm/imagereward","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"imagereward-learning-and-evaluating-human-1","repo_url":"https://github.com/AMD-AIG-AIMA/ReNeg","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"imagereward-learning-and-evaluating-human-1","repo_url":"https://github.com/LemonTwoL/ReNeg","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"image-generation","task_name":"Image Generation"},{"task_slug":"preference-mapping","task_name":"Preference Mapping"},{"task_slug":"text-to-image-generation-1","task_name":"Text to Image Generation"},{"task_slug":"text-to-image-generation","task_name":"Text-to-Image Generation"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2304.05977","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2304.05977"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/AMD-AIG-AIMA/ReNeg","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/thudm/imagereward","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/LemonTwoL/ReNeg","reach":null},{"provenance":"deterministic:regex_extraction","url":"https://github.com/THUDM/ImageReward","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_fixture":1,"ran_violates":1,"ran":1,"unverified":9},"by_repo_kind":{"official":{"samples":10,"ran":2,"repositories":1},"listed":{"samples":2,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":2,"samples":[{"code_sha256_prefix":"d91265552bd9c879","entry":"get_reward_fn","repo":"LemonTwoL/ReNeg","repo_kind":"listed","path":"reward_fn.py","file_url":"https://github.com/LemonTwoL/ReNeg/blob/HEAD/reward_fn.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"DEP_MISSING","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d91265552bd9c879"}},{"code_sha256_prefix":"4d1d9c47619920ba","entry":"is_url","repo":"THUDM/ImageReward","repo_kind":"official","path":"ImageReward/models/BLIP/blip.py","file_url":"https://github.com/THUDM/ImageReward/blob/HEAD/ImageReward/models/BLIP/blip.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4d1d9c47619920ba"}},{"code_sha256_prefix":"96b66ffedde07222","entry":"load","repo":"THUDM/ImageReward","repo_kind":"official","path":"ImageReward/utils.py","file_url":"https://github.com/THUDM/ImageReward/blob/HEAD/ImageReward/utils.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"96b66ffedde07222"}},{"code_sha256_prefix":"b1ae6f0bdf766e6c","entry":"ImageReward_download","repo":"THUDM/ImageReward","repo_kind":"official","path":"ImageReward/utils.py","file_url":"https://github.com/THUDM/ImageReward/blob/HEAD/ImageReward/utils.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b1ae6f0bdf766e6c"}},{"code_sha256_prefix":"f837c334ddfa8af4","entry":"create_vit","repo":"THUDM/ImageReward","repo_kind":"official","path":"ImageReward/models/BLIP/blip.py","file_url":"https://github.com/THUDM/ImageReward/blob/HEAD/ImageReward/models/BLIP/blip.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f837c334ddfa8af4"}},{"code_sha256_prefix":"69fa0b8db2b6b3de","entry":"encode_prompt_rm","repo":"THUDM/ImageReward","repo_kind":"official","path":"ImageReward/ReFL_SDXL.py","file_url":"https://github.com/THUDM/ImageReward/blob/HEAD/ImageReward/ReFL_SDXL.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"69fa0b8db2b6b3de"}},{"code_sha256_prefix":"6b8ba3c0c865429c","entry":"get_hpsv2_fn","repo":"LemonTwoL/ReNeg","repo_kind":"listed","path":"reward_fn.py","file_url":"https://github.com/LemonTwoL/ReNeg/blob/HEAD/reward_fn.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6b8ba3c0c865429c"}},{"code_sha256_prefix":"01e3b39e9e10abfb","entry":"import_model_class_from_model_name_or_path","repo":"THUDM/ImageReward","repo_kind":"official","path":"ImageReward/ReFL_SDXL.py","file_url":"https://github.com/THUDM/ImageReward/blob/HEAD/ImageReward/ReFL_SDXL.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"01e3b39e9e10abfb"}},{"code_sha256_prefix":"8337576ab7f136c1","entry":"load_checkpoint","repo":"THUDM/ImageReward","repo_kind":"official","path":"ImageReward/models/BLIP/blip.py","file_url":"https://github.com/THUDM/ImageReward/blob/HEAD/ImageReward/models/BLIP/blip.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8337576ab7f136c1"}},{"code_sha256_prefix":"ef6e5967acc37116","entry":"load_score","repo":"THUDM/ImageReward","repo_kind":"official","path":"ImageReward/utils.py","file_url":"https://github.com/THUDM/ImageReward/blob/HEAD/ImageReward/utils.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ef6e5967acc37116"}},{"code_sha256_prefix":"f4d1dbd44db71bc0","entry":"parse_args","repo":"THUDM/ImageReward","repo_kind":"official","path":"ImageReward/ReFL_SDXL.py","file_url":"https://github.com/THUDM/ImageReward/blob/HEAD/ImageReward/ReFL_SDXL.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f4d1dbd44db71bc0"}},{"code_sha256_prefix":"e8c37411989e89c1","entry":"parse_args","repo":"THUDM/ImageReward","repo_kind":"official","path":"ImageReward/ReFL_SDXL_LoRA.py","file_url":"https://github.com/THUDM/ImageReward/blob/HEAD/ImageReward/ReFL_SDXL_LoRA.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e8c37411989e89c1"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}