{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/spa-3d-spatial-awareness-enables-effective","title":"SPA: 3D Spatial-Awareness Enables Effective Embodied Representation","arxiv_id":"2410.08208","date":"2024-10-10","proceeding":null,"authors":["Haoyi Zhu","Honghui Yang","Yating Wang","Jiange Yang","LiMin Wang","Tong He"],"abstract":"In this paper, we introduce SPA, a novel representation learning framework that emphasizes the importance of 3D spatial awareness in embodied AI. Our approach leverages differentiable neural rendering on multi-view images to endow a vanilla Vision Transformer (ViT) with intrinsic spatial understanding. We present the most comprehensive evaluation of embodied representation learning to date, covering 268 tasks across 8 simulators with diverse policies in both single-task and language-conditioned multi-task scenarios. The results are compelling: SPA consistently outperforms more than 10 state-of-the-art representation methods, including those specifically designed for embodied AI, vision-centric tasks, and multi-modal applications, while using less training data. Furthermore, we conduct a series of real-world experiments to confirm its effectiveness in practical scenarios. These results highlight the critical role of 3D spatial awareness for embodied representation learning. Our strongest model takes more than 6000 GPU hours to train and we are committed to open-sourcing all code and model weights to foster future research in embodied representation learning. Project Page: https://haoyizhu.github.io/spa/.","url_abs":"https://arxiv.org/abs/2410.08208v3","url_pdf":"https://arxiv.org/pdf/2410.08208v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"spa-3d-spatial-awareness-enables-effective","repo_url":"https://github.com/haoyizhu/realrobot","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":null,"task_name":"GPU"},{"task_slug":"neural-rendering","task_name":"Neural Rendering"},{"task_slug":"representation-learning","task_name":"Representation Learning"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"},{"method_slug":"vision-transformer","method_name":"Vision Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2410.08208","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.08208"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/haoyizhu/realrobot","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":5,"unverified":1},"by_repo_kind":{"official":{"samples":6,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"a488db5b59ecea96","entry":"check_state_dict","repo":"haoyizhu/realrobot","repo_kind":"official","path":"src/utils/common_utils.py","file_url":"https://github.com/haoyizhu/realrobot/blob/HEAD/src/utils/common_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a488db5b59ecea96"}},{"code_sha256_prefix":"d9118aea6a0811f4","entry":"load_bytes","repo":"haoyizhu/realrobot","repo_kind":"official","path":"src/utils/io.py","file_url":"https://github.com/haoyizhu/realrobot/blob/HEAD/src/utils/io.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d9118aea6a0811f4"}},{"code_sha256_prefix":"7843f915d988dd5e","entry":"load_numpy_text","repo":"haoyizhu/realrobot","repo_kind":"official","path":"src/utils/io.py","file_url":"https://github.com/haoyizhu/realrobot/blob/HEAD/src/utils/io.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7843f915d988dd5e"}},{"code_sha256_prefix":"76e8fea908592e80","entry":"load_text","repo":"haoyizhu/realrobot","repo_kind":"official","path":"src/utils/io.py","file_url":"https://github.com/haoyizhu/realrobot/blob/HEAD/src/utils/io.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"76e8fea908592e80"}},{"code_sha256_prefix":"0d86269cf8e78d35","entry":"reduce_mean","repo":"haoyizhu/realrobot","repo_kind":"official","path":"src/utils/common_utils.py","file_url":"https://github.com/haoyizhu/realrobot/blob/HEAD/src/utils/common_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0d86269cf8e78d35"}},{"code_sha256_prefix":"f7898a5762a4d3e4","entry":"multi_apply","repo":"haoyizhu/realrobot","repo_kind":"official","path":"src/utils/common_utils.py","file_url":"https://github.com/haoyizhu/realrobot/blob/HEAD/src/utils/common_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f7898a5762a4d3e4"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}