{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/zero-shot-robotic-manipulation-with","title":"Zero-Shot Robotic Manipulation with Pretrained Image-Editing Diffusion Models","arxiv_id":"2310.10639","date":"2023-10-16","proceeding":null,"authors":["Kevin Black","Mitsuhiko Nakamoto","Pranav Atreya","Homer Walke","Chelsea Finn","Aviral Kumar","Sergey Levine"],"abstract":"If generalist robots are to operate in truly unstructured environments, they need to be able to recognize and reason about novel objects and scenarios. Such objects and scenarios might not be present in the robot's own training data. We propose SuSIE, a method that leverages an image-editing diffusion model to act as a high-level planner by proposing intermediate subgoals that a low-level controller can accomplish. Specifically, we finetune InstructPix2Pix on video data, consisting of both human videos and robot rollouts, such that it outputs hypothetical future \"subgoal\" observations given the robot's current observation and a language command. We also use the robot data to train a low-level goal-conditioned policy to act as the aforementioned low-level controller. We find that the high-level subgoal predictions can utilize Internet-scale pretraining and visual understanding to guide the low-level goal-conditioned policy, achieving significantly better generalization and precision than conventional language-conditioned policies. We achieve state-of-the-art results on the CALVIN benchmark, and also demonstrate robust generalization on real-world manipulation tasks, beating strong baselines that have access to privileged information or that utilize orders of magnitude more compute and training data. The project website can be found at http://rail-berkeley.github.io/susie .","url_abs":"https://arxiv.org/abs/2310.10639v1","url_pdf":"https://arxiv.org/pdf/2310.10639v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"links_only","authors_date_abstract":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license), from the Kaggle arXiv metadata snapshot of 2026-09-12"},"code_links":[{"paper_slug":"zero-shot-robotic-manipulation-with","repo_url":"https://github.com/kvablack/susie","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"jax","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2310.10639","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.10639"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kvablack/susie","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":5,"unverified":6},"by_repo_kind":{"official":{"samples":11,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"190dbaa1670129c4","entry":"host_broadcast_str","repo":"kvablack/susie","repo_kind":"official","path":"susie/jax_utils.py","file_url":"https://github.com/kvablack/susie/blob/HEAD/susie/jax_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"190dbaa1670129c4"}},{"code_sha256_prefix":"e41d41d8c82debdb","entry":"linear_log_snr","repo":"kvablack/susie","repo_kind":"official","path":"susie/scheduling.py","file_url":"https://github.com/kvablack/susie/blob/HEAD/susie/scheduling.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e41d41d8c82debdb"}},{"code_sha256_prefix":"96fba00a5828faa6","entry":"lnpoch","repo":"kvablack/susie","repo_kind":"official","path":"susie/scheduling.py","file_url":"https://github.com/kvablack/susie/blob/HEAD/susie/scheduling.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"96fba00a5828faa6"}},{"code_sha256_prefix":"d07fa6edf49d1c51","entry":"q_sample","repo":"kvablack/susie","repo_kind":"official","path":"susie/sampling.py","file_url":"https://github.com/kvablack/susie/blob/HEAD/susie/sampling.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d07fa6edf49d1c51"}},{"code_sha256_prefix":"61299d25063566fa","entry":"scaled_linear_log_snr","repo":"kvablack/susie","repo_kind":"official","path":"susie/scheduling.py","file_url":"https://github.com/kvablack/susie/blob/HEAD/susie/scheduling.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"61299d25063566fa"}},{"code_sha256_prefix":"b6e22297f43ef430","entry":"get_config","repo":"kvablack/susie","repo_kind":"official","path":"configs/base.py","file_url":"https://github.com/kvablack/susie/blob/HEAD/configs/base.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b6e22297f43ef430"}},{"code_sha256_prefix":"98527c72d7413f40","entry":"load_text_encoder","repo":"kvablack/susie","repo_kind":"official","path":"susie/model.py","file_url":"https://github.com/kvablack/susie/blob/HEAD/susie/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"98527c72d7413f40"}},{"code_sha256_prefix":"27978fad4ba438e0","entry":"model_predict","repo":"kvablack/susie","repo_kind":"official","path":"susie/sampling.py","file_url":"https://github.com/kvablack/susie/blob/HEAD/susie/sampling.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"27978fad4ba438e0"}},{"code_sha256_prefix":"5f121436fc522bca","entry":"replicate","repo":"kvablack/susie","repo_kind":"official","path":"susie/jax_utils.py","file_url":"https://github.com/kvablack/susie/blob/HEAD/susie/jax_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5f121436fc522bca"}},{"code_sha256_prefix":"8f36708327de063d","entry":"sample_step","repo":"kvablack/susie","repo_kind":"official","path":"susie/sampling.py","file_url":"https://github.com/kvablack/susie/blob/HEAD/susie/sampling.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8f36708327de063d"}},{"code_sha256_prefix":"d4f83d2a17ad94bf","entry":"shard_along_first_axis","repo":"kvablack/susie","repo_kind":"official","path":"susie/jax_utils.py","file_url":"https://github.com/kvablack/susie/blob/HEAD/susie/jax_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d4f83d2a17ad94bf"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}