{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/3d-vista-pre-trained-transformer-for-3d","title":"3D-VisTA: Pre-trained Transformer for 3D Vision and Text Alignment","arxiv_id":"2308.04352","date":"2023-08-08","proceeding":"ICCV 2023 1","authors":["Ziyu Zhu","Xiaojian Ma","Yixin Chen","Zhidong Deng","Siyuan Huang","Qing Li"],"abstract":"3D vision-language grounding (3D-VL) is an emerging field that aims to connect the 3D physical world with natural language, which is crucial for achieving embodied intelligence. Current 3D-VL models rely heavily on sophisticated modules, auxiliary losses, and optimization tricks, which calls for a simple and unified model. In this paper, we propose 3D-VisTA, a pre-trained Transformer for 3D Vision and Text Alignment that can be easily adapted to various downstream tasks. 3D-VisTA simply utilizes self-attention layers for both single-modal modeling and multi-modal fusion without any sophisticated task-specific design. To further enhance its performance on 3D-VL tasks, we construct ScanScribe, the first large-scale 3D scene-text pairs dataset for 3D-VL pre-training. ScanScribe contains 2,995 RGB-D scans for 1,185 unique indoor scenes originating from ScanNet and 3R-Scan datasets, along with paired 278K scene descriptions generated from existing 3D-VL tasks, templates, and GPT-3. 3D-VisTA is pre-trained on ScanScribe via masked language/object modeling and scene-text matching. It achieves state-of-the-art results on various 3D-VL tasks, ranging from visual grounding and dense captioning to question answering and situated reasoning. Moreover, 3D-VisTA demonstrates superior data efficiency, obtaining strong performance even with limited annotations during downstream task fine-tuning.","url_abs":"https://arxiv.org/abs/2308.04352v1","url_pdf":"https://arxiv.org/pdf/2308.04352v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"3d-vista-pre-trained-transformer-for-3d","repo_url":"https://github.com/3d-vista/3D-VisTA","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"3d-question-answering-3d-qa","task_name":"3D Question Answering (3D-QA)"},{"task_slug":"dense-captioning","task_name":"Dense Captioning"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"text-matching","task_name":"Text Matching"},{"task_slug":"visual-grounding","task_name":"Visual Grounding"},{"task_slug":"visual-reasoning","task_name":"Visual Reasoning"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"cosine-annealing","method_name":"Cosine Annealing"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt-3","method_name":"GPT-3"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-cosine-annealing","method_name":"Linear Warmup With Cosine Annealing"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"},{"method_slug":"weight-decay","method_name":"Weight Decay"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/3d-question-answering-3d-qa-on-sqa3d","task":"3D Question Answering (3D-QA)","dataset":"SQA3D","model":"3D-VisTA","rank_in_archive_order":8,"of":13,"metrics":{"Exact Match":"48.5"},"uses_additional_data":false},{"leaderboard":"/sota/3d-question-answering-3d-qa-on-scanqa-test-w","task":"3D Question Answering (3D-QA)","dataset":"ScanQA Test w/ objects","model":"3D-VisTA","rank_in_archive_order":9,"of":18,"metrics":{"BLEU-1":"-","BLEU-4":"10.4","CIDEr":"69.6","Exact Match":"22.4","METEOR":"13.9","ROUGE":"35.7"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2308.04352","atlas_url":"https://app.syntology.ai/?focus=2308.04352","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2308.04352"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/3d-vista/3D-VisTA","reach":null}],"summary":{"ran":2,"ran_violates":1,"ran_honours":1,"unverified":2},"by_repo_kind":{"listed":{"samples":6,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"af392075ecc51271","entry":"Registry","repo":"3d-vista/3D-VisTA","repo_kind":"listed","path":"model/vision/unified_encoder.py","file_url":"https://github.com/3d-vista/3D-VisTA/blob/HEAD/model/vision/unified_encoder.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"af392075ecc51271"}},{"code_sha256_prefix":"8f3dbe47a6eb7b40","entry":"TransformerEncoderLayer","repo":"3d-vista/3D-VisTA","repo_kind":"listed","path":"model/vision/unified_encoder.py","file_url":"https://github.com/3d-vista/3D-VisTA/blob/HEAD/model/vision/unified_encoder.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8f3dbe47a6eb7b40"}},{"code_sha256_prefix":"ef7878a2c111e5e2","entry":"generate_causal_mask","repo":"3d-vista/3D-VisTA","repo_kind":"listed","path":"model/vision/unified_encoder.py","file_url":"https://github.com/3d-vista/3D-VisTA/blob/HEAD/model/vision/unified_encoder.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ef7878a2c111e5e2"}},{"code_sha256_prefix":"e4b5ceca275ea601","entry":"generate_mm_casual_mask","repo":"3d-vista/3D-VisTA","repo_kind":"listed","path":"model/vision/unified_encoder.py","file_url":"https://github.com/3d-vista/3D-VisTA/blob/HEAD/model/vision/unified_encoder.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e4b5ceca275ea601"}},{"code_sha256_prefix":"5a6f1742cb13f1fc","entry":"UnifiedSpatialCrossEncoderV2","repo":"3d-vista/3D-VisTA","repo_kind":"listed","path":"model/vision/unified_encoder.py","file_url":"https://github.com/3d-vista/3D-VisTA/blob/HEAD/model/vision/unified_encoder.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5a6f1742cb13f1fc"}},{"code_sha256_prefix":"6c816339122f69e9","entry":"init_weights","repo":"3d-vista/3D-VisTA","repo_kind":"listed","path":"model/vision/unified_encoder.py","file_url":"https://github.com/3d-vista/3D-VisTA/blob/HEAD/model/vision/unified_encoder.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6c816339122f69e9"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}