{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/weakly-supervised-3d-open-vocabulary-1","title":"Weakly Supervised 3D Open-vocabulary Segmentation","arxiv_id":"2305.14093","date":"2023-05-23","proceeding":"NeurIPS 2023 11","authors":["Kunhao Liu","Fangneng Zhan","Jiahui Zhang","Muyu Xu","Yingchen Yu","Abdulmotaleb El Saddik","Christian Theobalt","Eric Xing","Shijian Lu"],"abstract":"Open-vocabulary segmentation of 3D scenes is a fundamental function of human perception and thus a crucial objective in computer vision research. However, this task is heavily impeded by the lack of large-scale and diverse 3D open-vocabulary segmentation datasets for training robust and generalizable models. Distilling knowledge from pre-trained 2D open-vocabulary segmentation models helps but it compromises the open-vocabulary feature as the 2D models are mostly finetuned with close-vocabulary datasets. We tackle the challenges in 3D open-vocabulary segmentation by exploiting pre-trained foundation models CLIP and DINO in a weakly supervised manner. Specifically, given only the open-vocabulary text descriptions of the objects in a scene, we distill the open-vocabulary multimodal knowledge and object reasoning capability of CLIP and DINO into a neural radiance field (NeRF), which effectively lifts 2D features into view-consistent 3D segmentation. A notable aspect of our approach is that it does not require any manual segmentation annotations for either the foundation models or the distillation process. Extensive experiments show that our method even outperforms fully supervised models trained with segmentation annotations in certain scenes, suggesting that 3D open-vocabulary segmentation can be effectively learned from 2D images and text-image pairs. Code is available at \\url{https://github.com/Kunhao-Liu/3D-OVS}.","url_abs":"https://arxiv.org/abs/2305.14093v4","url_pdf":"https://arxiv.org/pdf/2305.14093v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"weakly-supervised-3d-open-vocabulary-1","repo_url":"https://github.com/kunhao-liu/3d-ovs","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"nerf","task_name":"NeRF"},{"task_slug":"segmentation","task_name":"Segmentation"}],"methods":[{"method_slug":"attention","method_name":"Attention"},{"method_slug":"clip","method_name":"CLIP"},{"method_slug":"dino","method_name":"DINO"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"vision-transformer","method_name":"Vision Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2305.14093","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.14093"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/Kunhao-Liu/3D-OVS","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kunhao-liu/3d-ovs","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":9,"ran_fixture":1,"ran_draft_wrong":1,"unverified":3},"by_repo_kind":{"official":{"samples":14,"ran":11,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"945933374c5e3cbf","entry":"OctreeRender_trilinear_fast","repo":"Kunhao-Liu/3D-OVS","repo_kind":"official","path":"renderer.py","file_url":"https://github.com/Kunhao-Liu/3D-OVS/blob/HEAD/renderer.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"945933374c5e3cbf"}},{"code_sha256_prefix":"eae018b803b70a00","entry":"OctreeRender_trilinear_fast_depth","repo":"Kunhao-Liu/3D-OVS","repo_kind":"official","path":"renderer.py","file_url":"https://github.com/Kunhao-Liu/3D-OVS/blob/HEAD/renderer.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"eae018b803b70a00"}},{"code_sha256_prefix":"db95ecdf5d4abe6b","entry":"eval_sh","repo":"Kunhao-Liu/3D-OVS","repo_kind":"official","path":"models/sh.py","file_url":"https://github.com/Kunhao-Liu/3D-OVS/blob/HEAD/models/sh.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"db95ecdf5d4abe6b"}},{"code_sha256_prefix":"caa9e209a9083a0f","entry":"eval_sh_bases","repo":"Kunhao-Liu/3D-OVS","repo_kind":"official","path":"models/sh.py","file_url":"https://github.com/Kunhao-Liu/3D-OVS/blob/HEAD/models/sh.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"caa9e209a9083a0f"}},{"code_sha256_prefix":"dc510fe4f2261490","entry":"init_log","repo":"Kunhao-Liu/3D-OVS","repo_kind":"official","path":"funcs.py","file_url":"https://github.com/Kunhao-Liu/3D-OVS/blob/HEAD/funcs.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dc510fe4f2261490"}},{"code_sha256_prefix":"11a10efa60767162","entry":"positional_encoding","repo":"Kunhao-Liu/3D-OVS","repo_kind":"official","path":"models/tensorBase.py","file_url":"https://github.com/Kunhao-Liu/3D-OVS/blob/HEAD/models/tensorBase.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"11a10efa60767162"}},{"code_sha256_prefix":"c1acffa981204fee","entry":"raw2alpha","repo":"Kunhao-Liu/3D-OVS","repo_kind":"official","path":"models/tensorBase.py","file_url":"https://github.com/Kunhao-Liu/3D-OVS/blob/HEAD/models/tensorBase.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c1acffa981204fee"}},{"code_sha256_prefix":"521893f959a2ef80","entry":"sharpness","repo":"Kunhao-Liu/3D-OVS","repo_kind":"official","path":"dataLoader/colmap2nerf.py","file_url":"https://github.com/Kunhao-Liu/3D-OVS/blob/HEAD/dataLoader/colmap2nerf.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"521893f959a2ef80"}},{"code_sha256_prefix":"6ccb6c10eee60ba1","entry":"variance_of_laplacian","repo":"Kunhao-Liu/3D-OVS","repo_kind":"official","path":"dataLoader/colmap2nerf.py","file_url":"https://github.com/Kunhao-Liu/3D-OVS/blob/HEAD/dataLoader/colmap2nerf.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6ccb6c10eee60ba1"}},{"code_sha256_prefix":"5346431312bff754","entry":"visualize_depth","repo":"Kunhao-Liu/3D-OVS","repo_kind":"official","path":"funcs.py","file_url":"https://github.com/Kunhao-Liu/3D-OVS/blob/HEAD/funcs.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5346431312bff754"}},{"code_sha256_prefix":"c775698c33cfa2de","entry":"visualize_depth_numpy","repo":"Kunhao-Liu/3D-OVS","repo_kind":"official","path":"funcs.py","file_url":"https://github.com/Kunhao-Liu/3D-OVS/blob/HEAD/funcs.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c775698c33cfa2de"}},{"code_sha256_prefix":"3a530f556e40d9c0","entry":"SHRender","repo":"Kunhao-Liu/3D-OVS","repo_kind":"official","path":"models/tensorBase.py","file_url":"https://github.com/Kunhao-Liu/3D-OVS/blob/HEAD/models/tensorBase.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3a530f556e40d9c0"}},{"code_sha256_prefix":"9a9881451ca96ede","entry":"evaluation","repo":"Kunhao-Liu/3D-OVS","repo_kind":"official","path":"renderer.py","file_url":"https://github.com/Kunhao-Liu/3D-OVS/blob/HEAD/renderer.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9a9881451ca96ede"}},{"code_sha256_prefix":"6ce64cf0fbcd6bb1","entry":"qvec2rotmat","repo":"Kunhao-Liu/3D-OVS","repo_kind":"official","path":"dataLoader/colmap2nerf.py","file_url":"https://github.com/Kunhao-Liu/3D-OVS/blob/HEAD/dataLoader/colmap2nerf.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6ce64cf0fbcd6bb1"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}