{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/tweediemix-improving-multi-concept-fusion-for","title":"TweedieMix: Improving Multi-Concept Fusion for Diffusion-based Image/Video Generation","arxiv_id":"2410.05591","date":"2024-10-08","proceeding":null,"authors":["Gihyun Kwon","Jong Chul Ye"],"abstract":"Despite significant advancements in customizing text-to-image and video generation models, generating images and videos that effectively integrate multiple personalized concepts remains a challenging task. To address this, we present TweedieMix, a novel method for composing customized diffusion models during the inference phase. By analyzing the properties of reverse diffusion sampling, our approach divides the sampling process into two stages. During the initial steps, we apply a multiple object-aware sampling technique to ensure the inclusion of the desired target objects. In the later steps, we blend the appearances of the custom concepts in the de-noised image space using Tweedie's formula. Our results demonstrate that TweedieMix can generate multiple personalized concepts with higher fidelity than existing methods. Moreover, our framework can be effortlessly extended to image-to-video diffusion models, enabling the generation of videos that feature multiple personalized concepts. Results and source code are in our anonymous project page.","url_abs":"https://arxiv.org/abs/2410.05591v1","url_pdf":"https://arxiv.org/pdf/2410.05591v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"tweediemix-improving-multi-concept-fusion-for","repo_url":"https://github.com/kwongihyun/tweediemix","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"video-generation","task_name":"Video Generation"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2410.05591","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.05591"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kwongihyun/tweediemix","reach":{"status":"ok"}}],"summary":{"ran":3,"ran_draft_wrong":1,"ran_honours":1,"unverified":6},"by_repo_kind":{"official":{"samples":11,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":11,"samples":[{"code_sha256_prefix":"31f5eddde2a66739","entry":"collate_fn","repo":"kwongihyun/tweediemix","repo_kind":"official","path":"concept_training/diffusers_data_pipeline_xl.py","file_url":"https://github.com/kwongihyun/tweediemix/blob/HEAD/concept_training/diffusers_data_pipeline_xl.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"31f5eddde2a66739"}},{"code_sha256_prefix":"34ea6b0541b391aa","entry":"create_lora_diffusion","repo":"kwongihyun/tweediemix","repo_kind":"official","path":"concept_training/diffusers_model_pipeline_xl_lora.py","file_url":"https://github.com/kwongihyun/tweediemix/blob/HEAD/concept_training/diffusers_model_pipeline_xl_lora.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"34ea6b0541b391aa"}},{"code_sha256_prefix":"59e2b0d52876e596","entry":"preprocess","repo":"kwongihyun/tweediemix","repo_kind":"official","path":"concept_training/diffusers_data_pipeline_xl.py","file_url":"https://github.com/kwongihyun/tweediemix/blob/HEAD/concept_training/diffusers_data_pipeline_xl.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"59e2b0d52876e596"}},{"code_sha256_prefix":"bea2d776a332f2b0","entry":"rescale_noise_cfg","repo":"kwongihyun/tweediemix","repo_kind":"official","path":"fusion_generation/fusion_sampling.py","file_url":"https://github.com/kwongihyun/tweediemix/blob/HEAD/fusion_generation/fusion_sampling.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bea2d776a332f2b0"}},{"code_sha256_prefix":"d3d9b5fd6b42f66c","entry":"tokenize_prompt","repo":"kwongihyun/tweediemix","repo_kind":"official","path":"fusion_generation/fusion_sampling.py","file_url":"https://github.com/kwongihyun/tweediemix/blob/HEAD/fusion_generation/fusion_sampling.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d3d9b5fd6b42f66c"}},{"code_sha256_prefix":"25e5625ebee05129","entry":"create_lora_diffusion_base","repo":"kwongihyun/tweediemix","repo_kind":"official","path":"concept_training/diffusers_model_pipeline_xl_lora.py","file_url":"https://github.com/kwongihyun/tweediemix/blob/HEAD/concept_training/diffusers_model_pipeline_xl_lora.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"25e5625ebee05129"}},{"code_sha256_prefix":"aaeaf6d5b92e765e","entry":"encode_prompt","repo":"kwongihyun/tweediemix","repo_kind":"official","path":"concept_training/diffusers_training_xl_lora.py","file_url":"https://github.com/kwongihyun/tweediemix/blob/HEAD/concept_training/diffusers_training_xl_lora.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"aaeaf6d5b92e765e"}},{"code_sha256_prefix":"6acfa115a97ea2a8","entry":"encode_prompt","repo":"kwongihyun/tweediemix","repo_kind":"official","path":"fusion_generation/fusion_sampling.py","file_url":"https://github.com/kwongihyun/tweediemix/blob/HEAD/fusion_generation/fusion_sampling.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6acfa115a97ea2a8"}},{"code_sha256_prefix":"86965b6bae051ea4","entry":"import_model_class_from_model_name_or_path","repo":"kwongihyun/tweediemix","repo_kind":"official","path":"concept_training/diffusers_training_xl_new.py","file_url":"https://github.com/kwongihyun/tweediemix/blob/HEAD/concept_training/diffusers_training_xl_new.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"86965b6bae051ea4"}},{"code_sha256_prefix":"b031d967bb7e5eb9","entry":"register_attention_control_efficient","repo":"kwongihyun/tweediemix","repo_kind":"official","path":"fusion_generation/utils_custom.py","file_url":"https://github.com/kwongihyun/tweediemix/blob/HEAD/fusion_generation/utils_custom.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b031d967bb7e5eb9"}},{"code_sha256_prefix":"ede96e6969a3affd","entry":"register_attention_control_efficient","repo":"kwongihyun/tweediemix","repo_kind":"official","path":"fusion_generation/utils_lora.py","file_url":"https://github.com/kwongihyun/tweediemix/blob/HEAD/fusion_generation/utils_lora.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ede96e6969a3affd"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}