{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/controlnext-powerful-and-efficient-control","title":"ControlNeXt: Powerful and Efficient Control for Image and Video Generation","arxiv_id":"2408.06070","date":"2024-08-12","proceeding":null,"authors":["Bohao Peng","Jian Wang","Yuechen Zhang","Wenbo Li","Ming-Chang Yang","Jiaya Jia"],"abstract":"Diffusion models have demonstrated remarkable and robust abilities in both image and video generation. To achieve greater control over generated results, researchers introduce additional architectures, such as ControlNet, Adapters and ReferenceNet, to integrate conditioning controls. However, current controllable generation methods often require substantial additional computational resources, especially for video generation, and face challenges in training or exhibit weak control. In this paper, we propose ControlNeXt: a powerful and efficient method for controllable image and video generation. We first design a more straightforward and efficient architecture, replacing heavy additional branches with minimal additional cost compared to the base model. Such a concise structure also allows our method to seamlessly integrate with other LoRA weights, enabling style alteration without the need for additional training. As for training, we reduce up to 90% of learnable parameters compared to the alternatives. Furthermore, we propose another method called Cross Normalization (CN) as a replacement for Zero-Convolution' to achieve fast and stable training convergence. We have conducted various experiments with different base models across images and videos, demonstrating the robustness of our method.","url_abs":"https://arxiv.org/abs/2408.06070v2","url_pdf":"https://arxiv.org/pdf/2408.06070v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"controlnext-powerful-and-efficient-control","repo_url":"https://github.com/dvlab-research/controlnext","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"video-generation","task_name":"Video Generation"}],"methods":[{"method_slug":"base","method_name":"BASE"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2408.06070","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2408.06070"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/dvlab-research/controlnext","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_fixture":1,"ran":2,"unverified":6},"by_repo_kind":{"official":{"samples":9,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"bd5a05ce886da4ce","entry":"image_grid","repo":"dvlab-research/controlnext","repo_kind":"official","path":"ControlNeXt-SD1.5-Training/train_controlnext.py","file_url":"https://github.com/dvlab-research/controlnext/blob/HEAD/ControlNeXt-SD1.5-Training/train_controlnext.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bd5a05ce886da4ce"}},{"code_sha256_prefix":"52241d52a6acda50","entry":"parse_args","repo":"dvlab-research/controlnext","repo_kind":"official","path":"ControlNeXt-SD1.5-Training/run_controlnext.py","file_url":"https://github.com/dvlab-research/controlnext/blob/HEAD/ControlNeXt-SD1.5-Training/run_controlnext.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"52241d52a6acda50"}},{"code_sha256_prefix":"35717b791550ecb2","entry":"retrieve_timesteps","repo":"dvlab-research/controlnext","repo_kind":"official","path":"ControlNeXt-SD1.5/models/pipeline_controlnext.py","file_url":"https://github.com/dvlab-research/controlnext/blob/HEAD/ControlNeXt-SD1.5/models/pipeline_controlnext.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"35717b791550ecb2"}},{"code_sha256_prefix":"2f00b39db5a81466","entry":"import_model_class_from_model_name_or_path","repo":"dvlab-research/controlnext","repo_kind":"official","path":"ControlNeXt-SD1.5-Training/train_controlnext.py","file_url":"https://github.com/dvlab-research/controlnext/blob/HEAD/ControlNeXt-SD1.5-Training/train_controlnext.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2f00b39db5a81466"}},{"code_sha256_prefix":"86965b6bae051ea4","entry":"import_model_class_from_model_name_or_path","repo":"dvlab-research/controlnext","repo_kind":"official","path":"ControlNeXt-SDXL-Training/train_controlnext.py","file_url":"https://github.com/dvlab-research/controlnext/blob/HEAD/ControlNeXt-SDXL-Training/train_controlnext.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"86965b6bae051ea4"}},{"code_sha256_prefix":"1f631e56c6e042ac","entry":"import_model_class_from_model_name_or_path","repo":"dvlab-research/controlnext","repo_kind":"official","path":"ControlNeXt-SD1.5-Training/run_controlnext.py","file_url":"https://github.com/dvlab-research/controlnext/blob/HEAD/ControlNeXt-SD1.5-Training/run_controlnext.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1f631e56c6e042ac"}},{"code_sha256_prefix":"7f70a12d225a722c","entry":"parse_args","repo":"dvlab-research/controlnext","repo_kind":"official","path":"ControlNeXt-SD1.5/run_controlnext.py","file_url":"https://github.com/dvlab-research/controlnext/blob/HEAD/ControlNeXt-SD1.5/run_controlnext.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7f70a12d225a722c"}},{"code_sha256_prefix":"5473c01d472f643e","entry":"parse_args","repo":"dvlab-research/controlnext","repo_kind":"official","path":"ControlNeXt-SDXL-Training/train_controlnext.py","file_url":"https://github.com/dvlab-research/controlnext/blob/HEAD/ControlNeXt-SDXL-Training/train_controlnext.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5473c01d472f643e"}},{"code_sha256_prefix":"da94debb8019ad46","entry":"zero_module","repo":"dvlab-research/controlnext","repo_kind":"official","path":"ControlNeXt-SDXL-Training/models/controlnet.py","file_url":"https://github.com/dvlab-research/controlnext/blob/HEAD/ControlNeXt-SDXL-Training/models/controlnet.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"da94debb8019ad46"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}