{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/openscene-3d-scene-understanding-with-open","title":"OpenScene: 3D Scene Understanding with Open Vocabularies","arxiv_id":"2211.15654","date":"2022-11-28","proceeding":"CVPR 2023 1","authors":["Songyou Peng","Kyle Genova","Chiyu \"Max\" Jiang","Andrea Tagliasacchi","Marc Pollefeys","Thomas Funkhouser"],"abstract":"Traditional 3D scene understanding approaches rely on labeled 3D datasets to train a model for a single task with supervision. We propose OpenScene, an alternative approach where a model predicts dense features for 3D scene points that are co-embedded with text and image pixels in CLIP feature space. This zero-shot approach enables task-agnostic training and open-vocabulary queries. For example, to perform SOTA zero-shot 3D semantic segmentation it first infers CLIP features for every 3D point and later classifies them based on similarities to embeddings of arbitrary class labels. More interestingly, it enables a suite of open-vocabulary scene understanding applications that have never been done before. For example, it allows a user to enter an arbitrary text query and then see a heat map indicating which parts of a scene match. Our approach is effective at identifying objects, materials, affordances, activities, and room types in complex 3D scenes, all using a single model trained without any labeled 3D data.","url_abs":"https://arxiv.org/abs/2211.15654v2","url_pdf":"https://arxiv.org/pdf/2211.15654v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"openscene-3d-scene-understanding-with-open","repo_url":"https://github.com/pengsongyou/openscene","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"3d-open-vocabulary-instance-segmentation","task_name":"3D Open-Vocabulary Instance Segmentation"},{"task_slug":"3d-semantic-segmentation","task_name":"3D Semantic Segmentation"},{"task_slug":"scene-understanding","task_name":"Scene Understanding"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"}],"methods":[{"method_slug":"clip","method_name":"CLIP"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/3d-open-vocabulary-instance-segmentation-on-1","task":"3D Open-Vocabulary Instance Segmentation","dataset":"Replica","model":"OpenScene + Mask3D","rank_in_archive_order":7,"of":7,"metrics":{"mAP":"10.9"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2211.15654","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.15654"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/pengsongyou/openscene","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":4,"unverified":5},"by_repo_kind":{"listed":{"samples":9,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"f726974066cf1352","entry":"collation_fn","repo":"pengsongyou/openscene","repo_kind":"listed","path":"dataset/feature_loader.py","file_url":"https://github.com/pengsongyou/openscene/blob/HEAD/dataset/feature_loader.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f726974066cf1352"}},{"code_sha256_prefix":"65392c9cae62b77b","entry":"collation_fn_eval_all","repo":"pengsongyou/openscene","repo_kind":"listed","path":"dataset/feature_loader.py","file_url":"https://github.com/pengsongyou/openscene/blob/HEAD/dataset/feature_loader.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"65392c9cae62b77b"}},{"code_sha256_prefix":"cc9a3557465a7813","entry":"collation_fn_eval_all","repo":"pengsongyou/openscene","repo_kind":"listed","path":"dataset/point_loader.py","file_url":"https://github.com/pengsongyou/openscene/blob/HEAD/dataset/point_loader.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cc9a3557465a7813"}},{"code_sha256_prefix":"c4b88bc6388a332e","entry":"state_dict_remove_moudle","repo":"pengsongyou/openscene","repo_kind":"listed","path":"models/disnet.py","file_url":"https://github.com/pengsongyou/openscene/blob/HEAD/models/disnet.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c4b88bc6388a332e"}},{"code_sha256_prefix":"3095f89bf0fc823d","entry":"M","repo":"pengsongyou/openscene","repo_kind":"listed","path":"dataset/voxelizer.py","file_url":"https://github.com/pengsongyou/openscene/blob/HEAD/dataset/voxelizer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3095f89bf0fc823d"}},{"code_sha256_prefix":"bc5616d55b1b06b2","entry":"collation_fn","repo":"pengsongyou/openscene","repo_kind":"listed","path":"dataset/point_loader.py","file_url":"https://github.com/pengsongyou/openscene/blob/HEAD/dataset/point_loader.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bc5616d55b1b06b2"}},{"code_sha256_prefix":"0086c892bd1df796","entry":"fnv_hash_vec","repo":"pengsongyou/openscene","repo_kind":"listed","path":"dataset/voxelization_utils.py","file_url":"https://github.com/pengsongyou/openscene/blob/HEAD/dataset/voxelization_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0086c892bd1df796"}},{"code_sha256_prefix":"5905d8e45b03578e","entry":"get_release_scans","repo":"pengsongyou/openscene","repo_kind":"listed","path":"dataset/scannet/download-scannet.py","file_url":"https://github.com/pengsongyou/openscene/blob/HEAD/dataset/scannet/download-scannet.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5905d8e45b03578e"}},{"code_sha256_prefix":"afdc8ad15d9098b8","entry":"ravel_hash_vec","repo":"pengsongyou/openscene","repo_kind":"listed","path":"dataset/voxelization_utils.py","file_url":"https://github.com/pengsongyou/openscene/blob/HEAD/dataset/voxelization_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"afdc8ad15d9098b8"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}