{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/pointclip-v2-adapting-clip-for-powerful-3d","title":"PointCLIP V2: Prompting CLIP and GPT for Powerful 3D Open-world Learning","arxiv_id":"2211.11682","date":"2022-11-21","proceeding":"ICCV 2023 1","authors":["Xiangyang Zhu","Renrui Zhang","Bowei He","Ziyu Guo","Ziyao Zeng","Zipeng Qin","Shanghang Zhang","Peng Gao"],"abstract":"Large-scale pre-trained models have shown promising open-world performance for both vision and language tasks. However, their transferred capacity on 3D point clouds is still limited and only constrained to the classification task. In this paper, we first collaborate CLIP and GPT to be a unified 3D open-world learner, named as PointCLIP V2, which fully unleashes their potential for zero-shot 3D classification, segmentation, and detection. To better align 3D data with the pre-trained language knowledge, PointCLIP V2 contains two key designs. For the visual end, we prompt CLIP via a shape projection module to generate more realistic depth maps, narrowing the domain gap between projected point clouds with natural images. For the textual end, we prompt the GPT model to generate 3D-specific text as the input of CLIP's textual encoder. Without any training in 3D domains, our approach significantly surpasses PointCLIP by +42.90%, +40.44%, and +28.75% accuracy on three datasets for zero-shot 3D classification. On top of that, V2 can be extended to few-shot 3D classification, zero-shot 3D part segmentation, and 3D object detection in a simple manner, demonstrating our generalization ability for unified 3D open-world learning.","url_abs":"https://arxiv.org/abs/2211.11682v2","url_pdf":"https://arxiv.org/pdf/2211.11682v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"pointclip-v2-adapting-clip-for-powerful-3d","repo_url":"https://github.com/yangyangyang127/pointclip_v2","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"pointclip-v2-adapting-clip-for-powerful-3d","repo_url":"https://github.com/zrrskywalker/pointclip","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"3d-classification","task_name":"3D Classification"},{"task_slug":"3d-object-detection","task_name":"3D Object Detection"},{"task_slug":"3d-open-vocabulary-instance-segmentation","task_name":"3D Open-Vocabulary Instance Segmentation"},{"task_slug":"3d-part-segmentation","task_name":"3D Part Segmentation"},{"task_slug":"classification-1","task_name":"Classification"},{"task_slug":"descriptive","task_name":"Descriptive"},{"task_slug":"object-detection","task_name":"Object Detection"},{"task_slug":"open-vocabulary-object-detection","task_name":"Open Vocabulary Object Detection"},{"task_slug":"training-free-3d-part-segmentation","task_name":"Training-free 3D Part Segmentation"},{"task_slug":"training-free-3d-point-cloud-classification","task_name":"Training-free 3D Point Cloud Classification"},{"task_slug":"zero-shot-transfer-3d-point-cloud","task_name":"Zero-Shot Transfer 3D Point Cloud Classification"},{"task_slug":"zero-shot-3d-point-cloud-classification","task_name":"Zero-shot 3D Point Cloud Classification"},{"task_slug":"zero-shot-3d-classification","task_name":"Zero-shot 3D classification"},{"task_slug":"object-detection-1","task_name":"object-detection"}],"methods":[{"method_slug":"align","method_name":"ALIGN"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"clip","method_name":"CLIP"},{"method_slug":"cosine-annealing","method_name":"Cosine Annealing"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"discriminative-fine-tuning","method_name":"Discriminative Fine-Tuning"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt","method_name":"GPT"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-cosine-annealing","method_name":"Linear Warmup With Cosine Annealing"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/3d-open-vocabulary-instance-segmentation-on-3","task":"3D Open-Vocabulary Instance Segmentation","dataset":"STPLS3D","model":"PointCLIPV2","rank_in_archive_order":2,"of":3,"metrics":{"AP50":"03.1"},"uses_additional_data":false},{"leaderboard":"/sota/training-free-3d-part-segmentation-on","task":"Training-free 3D Part Segmentation","dataset":"ShapeNet-Part","model":"PointCLIP V2","rank_in_archive_order":2,"of":3,"metrics":{"Need 3D Data?":"No","mIoU":"48.4"},"uses_additional_data":true},{"leaderboard":"/sota/training-free-3d-point-cloud-classification","task":"Training-free 3D Point Cloud Classification","dataset":"ModelNet40","model":"PointCLIP V2","rank_in_archive_order":3,"of":7,"metrics":{"Accuracy (%)":"64.2","Need 3D Data?":"No"},"uses_additional_data":true},{"leaderboard":"/sota/training-free-3d-point-cloud-classification-1","task":"Training-free 3D Point Cloud Classification","dataset":"ScanObjectNN","model":"PointCLIP V2","rank_in_archive_order":3,"of":6,"metrics":{"Accuracy (%)":"35.4","Need 3D Data?":"No"},"uses_additional_data":true},{"leaderboard":"/sota/zero-shot-transfer-3d-point-cloud-1","task":"Zero-Shot Transfer 3D Point Cloud Classification","dataset":"ModelNet10","model":"PointCLIP V2","rank_in_archive_order":2,"of":4,"metrics":{"Accuracy (%)":"73.13"},"uses_additional_data":true},{"leaderboard":"/sota/zero-shot-transfer-3d-point-cloud","task":"Zero-Shot Transfer 3D Point Cloud Classification","dataset":"ModelNet40","model":"PointCLIP V2","rank_in_archive_order":11,"of":16,"metrics":{"Accuracy (%)":"64.22"},"uses_additional_data":true},{"leaderboard":"/sota/zero-shot-transfer-3d-point-cloud-2","task":"Zero-Shot Transfer 3D Point Cloud Classification","dataset":"ScanObjectNN","model":"PointCLIP V2","rank_in_archive_order":7,"of":10,"metrics":{"OBJ_BG Accuracy(%)":"41.22","OBJ_ONLY Accuracy(%)":"50.09","PB_T50_RS Accuracy (%)":"35.36"},"uses_additional_data":true}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2211.11682","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.11682"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/zrrskywalker/pointclip","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yangyangyang127/pointclip_v2","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":3,"ran_fixture":1,"ran_violates":1,"unverified":7},"by_repo_kind":{"official":{"samples":12,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"98f385d847636a3e","entry":"basic_clean","repo":"yangyangyang127/pointclip_v2","repo_kind":"official","path":"zeroshot_cls/clip/simple_tokenizer.py","file_url":"https://github.com/yangyangyang127/pointclip_v2/blob/HEAD/zeroshot_cls/clip/simple_tokenizer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"98f385d847636a3e"}},{"code_sha256_prefix":"c6c2758c0c2fa756","entry":"cal_loss","repo":"yangyangyang127/pointclip_v2","repo_kind":"official","path":"zeroshot_seg/util.py","file_url":"https://github.com/yangyangyang127/pointclip_v2/blob/HEAD/zeroshot_seg/util.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c6c2758c0c2fa756"}},{"code_sha256_prefix":"140122b970061273","entry":"euler2mat","repo":"yangyangyang127/pointclip_v2","repo_kind":"official","path":"zeroshot_seg/realistic_projection.py","file_url":"https://github.com/yangyangyang127/pointclip_v2/blob/HEAD/zeroshot_seg/realistic_projection.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"140122b970061273"}},{"code_sha256_prefix":"d919ae32e5e4e616","entry":"get_pairs","repo":"yangyangyang127/pointclip_v2","repo_kind":"official","path":"zeroshot_cls/clip/simple_tokenizer.py","file_url":"https://github.com/yangyangyang127/pointclip_v2/blob/HEAD/zeroshot_cls/clip/simple_tokenizer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d919ae32e5e4e616"}},{"code_sha256_prefix":"9542161e9640b858","entry":"whitespace_clean","repo":"yangyangyang127/pointclip_v2","repo_kind":"official","path":"zeroshot_cls/clip/simple_tokenizer.py","file_url":"https://github.com/yangyangyang127/pointclip_v2/blob/HEAD/zeroshot_cls/clip/simple_tokenizer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9542161e9640b858"}},{"code_sha256_prefix":"69c02e4eb1c02a52","entry":"build_model","repo":"yangyangyang127/pointclip_v2","repo_kind":"official","path":"zeroshot_cls/clip/model.py","file_url":"https://github.com/yangyangyang127/pointclip_v2/blob/HEAD/zeroshot_cls/clip/model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"69c02e4eb1c02a52"}},{"code_sha256_prefix":"e475fd539a310ed2","entry":"build_model","repo":"yangyangyang127/pointclip_v2","repo_kind":"official","path":"zeroshot_seg/clip/model.py","file_url":"https://github.com/yangyangyang127/pointclip_v2/blob/HEAD/zeroshot_seg/clip/model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e475fd539a310ed2"}},{"code_sha256_prefix":"d51a6e40c0aa5d4f","entry":"calculate_shape_IoU","repo":"yangyangyang127/pointclip_v2","repo_kind":"official","path":"zeroshot_seg/util.py","file_url":"https://github.com/yangyangyang127/pointclip_v2/blob/HEAD/zeroshot_seg/util.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d51a6e40c0aa5d4f"}},{"code_sha256_prefix":"9b099a224dbca79d","entry":"getGaussianKernel2D","repo":"yangyangyang127/pointclip_v2","repo_kind":"official","path":"zeroshot_seg/realistic_projection.py","file_url":"https://github.com/yangyangyang127/pointclip_v2/blob/HEAD/zeroshot_seg/realistic_projection.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9b099a224dbca79d"}},{"code_sha256_prefix":"fbf8c0143d9c48e3","entry":"load","repo":"yangyangyang127/pointclip_v2","repo_kind":"official","path":"zeroshot_cls/clip/clip.py","file_url":"https://github.com/yangyangyang127/pointclip_v2/blob/HEAD/zeroshot_cls/clip/clip.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fbf8c0143d9c48e3"}},{"code_sha256_prefix":"6ac515bebb5bb563","entry":"load_data_partseg","repo":"yangyangyang127/pointclip_v2","repo_kind":"official","path":"zeroshot_seg/data.py","file_url":"https://github.com/yangyangyang127/pointclip_v2/blob/HEAD/zeroshot_seg/data.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6ac515bebb5bb563"}},{"code_sha256_prefix":"706ac9d3b62c54e8","entry":"points2grid","repo":"yangyangyang127/pointclip_v2","repo_kind":"official","path":"zeroshot_seg/realistic_projection.py","file_url":"https://github.com/yangyangyang127/pointclip_v2/blob/HEAD/zeroshot_seg/realistic_projection.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"706ac9d3b62c54e8"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}