{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/vico-detail-preserving-visual-condition-for","title":"ViCo: Plug-and-play Visual Condition for Personalized Text-to-image Generation","arxiv_id":"2306.00971","date":"2023-06-01","proceeding":null,"authors":["Shaozhe Hao","Kai Han","Shihao Zhao","Kwan-Yee K. Wong"],"abstract":"Personalized text-to-image generation using diffusion models has recently emerged and garnered significant interest. This task learns a novel concept (e.g., a unique toy), illustrated in a handful of images, into a generative model that captures fine visual details and generates photorealistic images based on textual embeddings. In this paper, we present ViCo, a novel lightweight plug-and-play method that seamlessly integrates visual condition into personalized text-to-image generation. ViCo stands out for its unique feature of not requiring any fine-tuning of the original diffusion model parameters, thereby facilitating more flexible and scalable model deployment. This key advantage distinguishes ViCo from most existing models that necessitate partial or full diffusion fine-tuning. ViCo incorporates an image attention module that conditions the diffusion process on patch-wise visual semantics, and an attention-based object mask that comes at no extra cost from the attention module. Despite only requiring light parameter training (~6% compared to the diffusion U-Net), ViCo delivers performance that is on par with, or even surpasses, all state-of-the-art models, both qualitatively and quantitatively. This underscores the efficacy of ViCo, making it a highly promising solution for personalized text-to-image generation without the need for diffusion model fine-tuning. Code: https://github.com/haoosz/ViCo","url_abs":"https://arxiv.org/abs/2306.00971v2","url_pdf":"https://arxiv.org/pdf/2306.00971v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"vico-detail-preserving-visual-condition-for","repo_url":"https://github.com/haoosz/vico","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"image-generation","task_name":"Image Generation"},{"task_slug":"text-to-image-generation-1","task_name":"Text to Image Generation"},{"task_slug":"text-to-image-generation","task_name":"Text-to-Image Generation"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2306.00971","atlas_url":"https://app.syntology.ai/?focus=2306.00971","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.00971"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/haoosz/vico","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":5,"ran_violates":3,"ran_honours":2,"ran_fixture":1,"ran":3,"unverified":1},"by_repo_kind":{"official":{"samples":15,"ran":14,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"9fcdaa6e423e8aa7","entry":"Normalize","repo":"haoosz/vico","repo_kind":"official","path":"ldm/modules/diffusionmodules/model.py","file_url":"https://github.com/haoosz/vico/blob/HEAD/ldm/modules/diffusionmodules/model.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9fcdaa6e423e8aa7"}},{"code_sha256_prefix":"fe5dd5258046898c","entry":"always","repo":"haoosz/vico","repo_kind":"official","path":"ldm/modules/x_transformer.py","file_url":"https://github.com/haoosz/vico/blob/HEAD/ldm/modules/x_transformer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fe5dd5258046898c"}},{"code_sha256_prefix":"424012cb37b31172","entry":"default","repo":"haoosz/vico","repo_kind":"official","path":"ldm/modules/attention.py","file_url":"https://github.com/haoosz/vico/blob/HEAD/ldm/modules/attention.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"424012cb37b31172"}},{"code_sha256_prefix":"4cb732f513d69dfd","entry":"disabled_train","repo":"haoosz/vico","repo_kind":"official","path":"ldm/models/diffusion/classifier.py","file_url":"https://github.com/haoosz/vico/blob/HEAD/ldm/models/diffusion/classifier.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4cb732f513d69dfd"}},{"code_sha256_prefix":"aa5486a3650902d8","entry":"exists","repo":"haoosz/vico","repo_kind":"official","path":"ldm/modules/attention.py","file_url":"https://github.com/haoosz/vico/blob/HEAD/ldm/modules/attention.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"aa5486a3650902d8"}},{"code_sha256_prefix":"d5726f9e3a5684c3","entry":"get_bert_token_for_string","repo":"haoosz/vico","repo_kind":"official","path":"ldm/modules/embedding_manager.py","file_url":"https://github.com/haoosz/vico/blob/HEAD/ldm/modules/embedding_manager.py","link_basis":"plan_row","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d5726f9e3a5684c3"}},{"code_sha256_prefix":"41c6eb3a0aa3eb46","entry":"get_clip_token_for_string","repo":"haoosz/vico","repo_kind":"official","path":"ldm/modules/embedding_manager.py","file_url":"https://github.com/haoosz/vico/blob/HEAD/ldm/modules/embedding_manager.py","link_basis":"plan_row","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"41c6eb3a0aa3eb46"}},{"code_sha256_prefix":"cb49209c125de1b4","entry":"get_timestep_embedding","repo":"haoosz/vico","repo_kind":"official","path":"ldm/modules/diffusionmodules/model.py","file_url":"https://github.com/haoosz/vico/blob/HEAD/ldm/modules/diffusionmodules/model.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cb49209c125de1b4"}},{"code_sha256_prefix":"b1368330cf0f5642","entry":"isimage","repo":"haoosz/vico","repo_kind":"official","path":"ldm/util.py","file_url":"https://github.com/haoosz/vico/blob/HEAD/ldm/util.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b1368330cf0f5642"}},{"code_sha256_prefix":"d72762b700feee6f","entry":"ismap","repo":"haoosz/vico","repo_kind":"official","path":"ldm/util.py","file_url":"https://github.com/haoosz/vico/blob/HEAD/ldm/util.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d72762b700feee6f"}},{"code_sha256_prefix":"f41710299c49d36d","entry":"log_txt_as_img","repo":"haoosz/vico","repo_kind":"official","path":"ldm/util.py","file_url":"https://github.com/haoosz/vico/blob/HEAD/ldm/util.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f41710299c49d36d"}},{"code_sha256_prefix":"3137073275f8c21a","entry":"nonlinearity","repo":"haoosz/vico","repo_kind":"official","path":"ldm/modules/diffusionmodules/model.py","file_url":"https://github.com/haoosz/vico/blob/HEAD/ldm/modules/diffusionmodules/model.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3137073275f8c21a"}},{"code_sha256_prefix":"d48d8354986e3b0e","entry":"uniform_on_device","repo":"haoosz/vico","repo_kind":"official","path":"ldm/models/diffusion/ddpm.py","file_url":"https://github.com/haoosz/vico/blob/HEAD/ldm/models/diffusion/ddpm.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d48d8354986e3b0e"}},{"code_sha256_prefix":"9a299fe5ae09e407","entry":"uniq","repo":"haoosz/vico","repo_kind":"official","path":"ldm/modules/attention.py","file_url":"https://github.com/haoosz/vico/blob/HEAD/ldm/modules/attention.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9a299fe5ae09e407"}},{"code_sha256_prefix":"48a80bc1b4910426","entry":"get_embedding_for_clip_token","repo":"haoosz/vico","repo_kind":"official","path":"ldm/modules/embedding_manager.py","file_url":"https://github.com/haoosz/vico/blob/HEAD/ldm/modules/embedding_manager.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"48a80bc1b4910426"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}