{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/subject-driven-text-to-image-generation-via-1","title":"Subject-driven Text-to-Image Generation via Preference-based Reinforcement Learning","arxiv_id":"2407.12164","date":"2024-07-16","proceeding":null,"authors":["Yanting Miao","William Loh","Suraj Kothawade","Pascal Poupart","Abdullah Rashwan","Yeqing Li"],"abstract":"Text-to-image generative models have recently attracted considerable interest, enabling the synthesis of high-quality images from textual prompts. However, these models often lack the capability to generate specific subjects from given reference images or to synthesize novel renditions under varying conditions. Methods like DreamBooth and Subject-driven Text-to-Image (SuTI) have made significant progress in this area. Yet, both approaches primarily focus on enhancing similarity to reference images and require expensive setups, often overlooking the need for efficient training and avoiding overfitting to the reference images. In this work, we present the $\\lambda$-Harmonic reward function, which provides a reliable reward signal and enables early stopping for faster training and effective regularization. By combining the Bradley-Terry preference model, the $\\lambda$-Harmonic reward function also provides preference labels for subject-driven generation tasks. We propose Reward Preference Optimization (RPO), which offers a simpler setup (requiring only $3\\%$ of the negative samples used by DreamBooth) and fewer gradient steps for fine-tuning. Unlike most existing methods, our approach does not require training a text encoder or optimizing text embeddings and achieves text-image alignment by fine-tuning only the U-Net component. Empirically, $\\lambda$-Harmonic proves to be a reliable approach for model selection in subject-driven generation tasks. Based on preference labels and early stopping validation from the $\\lambda$-Harmonic reward function, our algorithm achieves a state-of-the-art CLIP-I score of 0.833 and a CLIP-T score of 0.314 on DreamBench.","url_abs":"https://arxiv.org/abs/2407.12164v3","url_pdf":"https://arxiv.org/pdf/2407.12164v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"subject-driven-text-to-image-generation-via-1","repo_url":"https://github.com/andrew-miao/RPO","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"image-generation","task_name":"Image Generation"},{"task_slug":"model-selection","task_name":"Model Selection"},{"task_slug":"text-to-image-generation-1","task_name":"Text to Image Generation"},{"task_slug":"text-to-image-generation","task_name":"Text-to-Image Generation"}],"methods":[{"method_slug":"concatenated-skip-connection","method_name":"Concatenated Skip Connection"},{"method_slug":"convolution","method_name":"Convolution"},{"method_slug":"early-stopping","method_name":"Early Stopping"},{"method_slug":"focus","method_name":"Focus"},{"method_slug":"max-pooling","method_name":"Max Pooling"},{"method_slug":"relu","method_name":"ReLU"},{"method_slug":"u-net","method_name":"U-Net"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2407.12164","atlas_url":"https://app.syntology.ai/?focus=2407.12164","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.12164"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/andrew-miao/RPO","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":4,"unverified":3},"by_repo_kind":{"official":{"samples":7,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"af6b60ee3678af1f","entry":"collate_fn","repo":"andrew-miao/RPO","repo_kind":"official","path":"train_sdxl.py","file_url":"https://github.com/andrew-miao/RPO/blob/HEAD/train_sdxl.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"af6b60ee3678af1f"}},{"code_sha256_prefix":"3c410fa9a055b554","entry":"cosine_similarity","repo":"andrew-miao/RPO","repo_kind":"official","path":"utils.py","file_url":"https://github.com/andrew-miao/RPO/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3c410fa9a055b554"}},{"code_sha256_prefix":"fd850f141d6a889a","entry":"read_images","repo":"andrew-miao/RPO","repo_kind":"official","path":"utils.py","file_url":"https://github.com/andrew-miao/RPO/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fd850f141d6a889a"}},{"code_sha256_prefix":"315891129682a491","entry":"remove_unique_token","repo":"andrew-miao/RPO","repo_kind":"official","path":"utils.py","file_url":"https://github.com/andrew-miao/RPO/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"315891129682a491"}},{"code_sha256_prefix":"8d53fe390a66ca77","entry":"CLIP_T_score","repo":"andrew-miao/RPO","repo_kind":"official","path":"evaluation_metrics.py","file_url":"https://github.com/andrew-miao/RPO/blob/HEAD/evaluation_metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8d53fe390a66ca77"}},{"code_sha256_prefix":"39503ff32cc75614","entry":"import_model_class_from_model_name_or_path","repo":"andrew-miao/RPO","repo_kind":"official","path":"train_sdxl.py","file_url":"https://github.com/andrew-miao/RPO/blob/HEAD/train_sdxl.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"39503ff32cc75614"}},{"code_sha256_prefix":"dfa2097dcc253eab","entry":"parse_args","repo":"andrew-miao/RPO","repo_kind":"official","path":"train_sdxl.py","file_url":"https://github.com/andrew-miao/RPO/blob/HEAD/train_sdxl.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dfa2097dcc253eab"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}