{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/robouniview-visual-language-model-with","title":"RoboUniView: Visual-Language Model with Unified View Representation for Robotic Manipulation","arxiv_id":"2406.18977","date":"2024-06-27","proceeding":null,"authors":["Fanfan Liu","Feng Yan","Liming Zheng","Chengjian Feng","Yiyang Huang","Lin Ma"],"abstract":"Utilizing Vision-Language Models (VLMs) for robotic manipulation represents a novel paradigm, aiming to enhance the model's ability to generalize to new objects and instructions. However, due to variations in camera specifications and mounting positions, existing methods exhibit significant performance disparities across different robotic platforms. To address this challenge, we propose RoboUniView in this paper, an innovative approach that decouples visual feature extraction from action learning. We first learn a unified view representation from multi-perspective views by pre-training on readily accessible data, and then derive actions from this unified view representation to control robotic manipulation. This unified view representation more accurately mirrors the physical world and is not constrained by the robotic platform's camera parameters. Thanks to this methodology, we achieve state-of-the-art performance on the demanding CALVIN benchmark, enhancing the success rate in the $D \\to D$ setting from 93.0% to 96.2%, and in the $ABC \\to D$ setting from 92.2% to 94.2%. Moreover, our model exhibits outstanding adaptability and flexibility: it maintains high performance under unseen camera parameters, can utilize multiple datasets with varying camera parameters, and is capable of joint cross-task learning across datasets. Code is provided for re-implementation. https://github.com/liufanfanlff/RoboUniview","url_abs":"https://arxiv.org/abs/2406.18977v3","url_pdf":"https://arxiv.org/pdf/2406.18977v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"robouniview-visual-language-model-with","repo_url":"https://github.com/liufanfanlff/robouniview","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"robot-manipulation","task_name":"Robot Manipulation"},{"task_slug":"zero-shot-generalization","task_name":"Zero-shot Generalization"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/robot-manipulation-on-calvin","task":"Robot Manipulation","dataset":"CALVIN","model":"RoboUniView","rank_in_archive_order":8,"of":19,"metrics":{"avg. sequence length (D to D)":"3.855"},"uses_additional_data":false},{"leaderboard":"/sota/zero-shot-generalization-on-calvin","task":"Zero-shot Generalization","dataset":"CALVIN","model":"RoboUniView","rank_in_archive_order":3,"of":5,"metrics":{"Avg. sequence length":"3.647"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2406.18977","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.18977"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/liufanfanlff/robouniview","reach":null}],"summary":{"ran_draft_wrong":1},"by_repo_kind":{"official":{"samples":1,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":1,"samples":[{"code_sha256_prefix":"55fc8f6fe8c8bc81","entry":"deproject","repo":"liufanfanlff/robouniview","repo_kind":"official","path":"robouniview/generate_data/generate_data.py","file_url":"https://github.com/liufanfanlff/robouniview/blob/HEAD/robouniview/generate_data/generate_data.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"55fc8f6fe8c8bc81"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}