{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/ingredients-blending-custom-photos-with-video","title":"Ingredients: Blending Custom Photos with Video Diffusion Transformers","arxiv_id":"2501.01790","date":"2025-01-03","proceeding":null,"authors":["Zhengcong Fei","Debang Li","Di Qiu","Changqian Yu","Mingyuan Fan"],"abstract":"This paper presents a powerful framework to customize video creations by incorporating multiple specific identity (ID) photos, with video diffusion Transformers, referred to as \\texttt{Ingredients}. Generally, our method consists of three primary modules: (\\textbf{i}) a facial extractor that captures versatile and precise facial features for each human ID from both global and local perspectives; (\\textbf{ii}) a multi-scale projector that maps face embeddings into the contextual space of image query in video diffusion transformers; (\\textbf{iii}) an ID router that dynamically combines and allocates multiple ID embedding to the corresponding space-time regions. Leveraging a meticulously curated text-video dataset and a multi-stage training protocol, \\texttt{Ingredients} demonstrates superior performance in turning custom photos into dynamic and personalized video content. Qualitative evaluations highlight the advantages of proposed method, positioning it as a significant advancement toward more effective generative video control tools in Transformer-based architecture, compared to existing methods. The data, code, and model weights are publicly available at: \\url{https://github.com/feizc/Ingredients}.","url_abs":"https://arxiv.org/abs/2501.01790v1","url_pdf":"https://arxiv.org/pdf/2501.01790v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"ingredients-blending-custom-photos-with-video","repo_url":"https://github.com/feizc/ingredients","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2501.01790","atlas_url":"https://app.syntology.ai/?focus=2501.01790","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2501.01790"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/feizc/ingredients","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_draft_wrong":2,"ran":9,"ran_honours":2,"ran_fixture":1,"unverified":8},"by_repo_kind":{"official":{"samples":22,"ran":14,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"5105747c2711b1cb","entry":"FeedForward","repo":"feizc/ingredients","repo_kind":"official","path":"models/local_facial_extractor.py","file_url":"https://github.com/feizc/ingredients/blob/HEAD/models/local_facial_extractor.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5105747c2711b1cb"}},{"code_sha256_prefix":"025423ed7b216773","entry":"compute_prompt_embeddings","repo":"feizc/ingredients","repo_kind":"official","path":"models/utils.py","file_url":"https://github.com/feizc/ingredients/blob/HEAD/models/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"025423ed7b216773"}},{"code_sha256_prefix":"85a6d411e524efe4","entry":"conv1x1","repo":"feizc/ingredients","repo_kind":"official","path":"metric/curricularface/model_resnet.py","file_url":"https://github.com/feizc/ingredients/blob/HEAD/metric/curricularface/model_resnet.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"85a6d411e524efe4"}},{"code_sha256_prefix":"6ccc5338f47a440c","entry":"conv3x3","repo":"feizc/ingredients","repo_kind":"official","path":"metric/curricularface/model_resnet.py","file_url":"https://github.com/feizc/ingredients/blob/HEAD/metric/curricularface/model_resnet.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6ccc5338f47a440c"}},{"code_sha256_prefix":"4ccfaec71f6cfcc8","entry":"draw_kps","repo":"feizc/ingredients","repo_kind":"official","path":"models/pipeline_ingredients.py","file_url":"https://github.com/feizc/ingredients/blob/HEAD/models/pipeline_ingredients.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4ccfaec71f6cfcc8"}},{"code_sha256_prefix":"ad5dbf57f3ea2633","entry":"get_block","repo":"feizc/ingredients","repo_kind":"official","path":"metric/curricularface/model_irse.py","file_url":"https://github.com/feizc/ingredients/blob/HEAD/metric/curricularface/model_irse.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ad5dbf57f3ea2633"}},{"code_sha256_prefix":"e82dfb7d2386721b","entry":"get_blocks","repo":"feizc/ingredients","repo_kind":"official","path":"metric/curricularface/model_irse.py","file_url":"https://github.com/feizc/ingredients/blob/HEAD/metric/curricularface/model_irse.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e82dfb7d2386721b"}},{"code_sha256_prefix":"e185497b8158057e","entry":"get_face_keypoints","repo":"feizc/ingredients","repo_kind":"official","path":"metric/face_sim_fid.py","file_url":"https://github.com/feizc/ingredients/blob/HEAD/metric/face_sim_fid.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e185497b8158057e"}},{"code_sha256_prefix":"734fd43f6aaef115","entry":"get_resize_crop_region_for_grid","repo":"feizc/ingredients","repo_kind":"official","path":"models/pipeline_ingredients.py","file_url":"https://github.com/feizc/ingredients/blob/HEAD/models/pipeline_ingredients.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"734fd43f6aaef115"}},{"code_sha256_prefix":"18703ab999a62c3e","entry":"load_image","repo":"feizc/ingredients","repo_kind":"official","path":"metric/face_sim_fid.py","file_url":"https://github.com/feizc/ingredients/blob/HEAD/metric/face_sim_fid.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"18703ab999a62c3e"}},{"code_sha256_prefix":"4cb2e2a2ca0bec9f","entry":"reshape_tensor","repo":"feizc/ingredients","repo_kind":"official","path":"models/local_facial_extractor.py","file_url":"https://github.com/feizc/ingredients/blob/HEAD/models/local_facial_extractor.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4cb2e2a2ca0bec9f"}},{"code_sha256_prefix":"22b1f260da28f6a5","entry":"retrieve_timesteps","repo":"feizc/ingredients","repo_kind":"official","path":"models/pipeline_cogvideox.py","file_url":"https://github.com/feizc/ingredients/blob/HEAD/models/pipeline_cogvideox.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"22b1f260da28f6a5"}},{"code_sha256_prefix":"8d3e6b254561c7ec","entry":"sample_video_frames","repo":"feizc/ingredients","repo_kind":"official","path":"metric/face_sim_fid.py","file_url":"https://github.com/feizc/ingredients/blob/HEAD/metric/face_sim_fid.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8d3e6b254561c7ec"}},{"code_sha256_prefix":"aabdf33f95d21edf","entry":"tensor_to_pil","repo":"feizc/ingredients","repo_kind":"official","path":"models/utils.py","file_url":"https://github.com/feizc/ingredients/blob/HEAD/models/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"aabdf33f95d21edf"}},{"code_sha256_prefix":"42efa8d0cbd120d8","entry":"IR_18","repo":"feizc/ingredients","repo_kind":"official","path":"metric/curricularface/model_irse.py","file_url":"https://github.com/feizc/ingredients/blob/HEAD/metric/curricularface/model_irse.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"42efa8d0cbd120d8"}},{"code_sha256_prefix":"d6583b15d8a7722f","entry":"ResNet_50","repo":"feizc/ingredients","repo_kind":"official","path":"metric/curricularface/model_resnet.py","file_url":"https://github.com/feizc/ingredients/blob/HEAD/metric/curricularface/model_resnet.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d6583b15d8a7722f"}},{"code_sha256_prefix":"7b654be5a00e575f","entry":"compute_clip_score","repo":"feizc/ingredients","repo_kind":"official","path":"metric/clip_score.py","file_url":"https://github.com/feizc/ingredients/blob/HEAD/metric/clip_score.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7b654be5a00e575f"}},{"code_sha256_prefix":"9ca4e9fb5eb8034d","entry":"encode_prompt","repo":"feizc/ingredients","repo_kind":"official","path":"models/utils.py","file_url":"https://github.com/feizc/ingredients/blob/HEAD/models/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9ca4e9fb5eb8034d"}},{"code_sha256_prefix":"5c7cf820e99443ff","entry":"get_model_config","repo":"feizc/ingredients","repo_kind":"official","path":"models/eva_clip/factory.py","file_url":"https://github.com/feizc/ingredients/blob/HEAD/models/eva_clip/factory.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5c7cf820e99443ff"}},{"code_sha256_prefix":"9b7c99fcbb58f26f","entry":"get_tokenizer","repo":"feizc/ingredients","repo_kind":"official","path":"models/eva_clip/factory.py","file_url":"https://github.com/feizc/ingredients/blob/HEAD/models/eva_clip/factory.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9b7c99fcbb58f26f"}},{"code_sha256_prefix":"ab854bdad372c33e","entry":"load_state_dict","repo":"feizc/ingredients","repo_kind":"official","path":"models/eva_clip/factory.py","file_url":"https://github.com/feizc/ingredients/blob/HEAD/models/eva_clip/factory.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ab854bdad372c33e"}},{"code_sha256_prefix":"c17ad34b7b0cba0e","entry":"process_image","repo":"feizc/ingredients","repo_kind":"official","path":"models/pipeline_ingredients.py","file_url":"https://github.com/feizc/ingredients/blob/HEAD/models/pipeline_ingredients.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c17ad34b7b0cba0e"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}