{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/multi3drefer-grounding-text-description-to","title":"Multi3DRefer: Grounding Text Description to Multiple 3D Objects","arxiv_id":"2309.05251","date":"2023-09-11","proceeding":"ICCV 2023 1","authors":["Yiming Zhang","ZeMing Gong","Angel X. Chang"],"abstract":"We introduce the task of localizing a flexible number of objects in real-world 3D scenes using natural language descriptions. Existing 3D visual grounding tasks focus on localizing a unique object given a text description. However, such a strict setting is unnatural as localizing potentially multiple objects is a common need in real-world scenarios and robotic tasks (e.g., visual navigation and object rearrangement). To address this setting we propose Multi3DRefer, generalizing the ScanRefer dataset and task. Our dataset contains 61926 descriptions of 11609 objects, where zero, single or multiple target objects are referenced by each description. We also introduce a new evaluation metric and benchmark methods from prior work to enable further investigation of multi-modal 3D scene understanding. Furthermore, we develop a better baseline leveraging 2D features from CLIP by rendering object proposals online with contrastive learning, which outperforms the state of the art on the ScanRefer benchmark.","url_abs":"https://arxiv.org/abs/2309.05251v1","url_pdf":"https://arxiv.org/pdf/2309.05251v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"multi3drefer-grounding-text-description-to","repo_url":"https://github.com/3dlg-hcvc/M3DRef-CLIP","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"3d-visual-grounding","task_name":"3D visual grounding"},{"task_slug":"contrastive-learning","task_name":"Contrastive Learning"},{"task_slug":"object","task_name":"Object"},{"task_slug":"object-rearrangement","task_name":"Object Rearrangement"},{"task_slug":"scene-understanding","task_name":"Scene Understanding"},{"task_slug":"visual-grounding","task_name":"Visual Grounding"},{"task_slug":"visual-navigation","task_name":"Visual Navigation"}],"methods":[{"method_slug":"clip","method_name":"CLIP"},{"method_slug":"focus","method_name":"Focus"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2309.05251","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2309.05251"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/3dlg-hcvc/M3DRef-CLIP","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":5,"unverified":3},"by_repo_kind":{"official":{"samples":8,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"fad251c358d62da1","entry":"decode_stimulus_string","repo":"3dlg-hcvc/M3DRef-CLIP","repo_kind":"official","path":"dataset/nr3d/add_evaluation_labels.py","file_url":"https://github.com/3dlg-hcvc/M3DRef-CLIP/blob/HEAD/dataset/nr3d/add_evaluation_labels.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fad251c358d62da1"}},{"code_sha256_prefix":"3c337a75a8d80a18","entry":"get_easy_mask","repo":"3dlg-hcvc/M3DRef-CLIP","repo_kind":"official","path":"dataset/nr3d/add_evaluation_labels.py","file_url":"https://github.com/3dlg-hcvc/M3DRef-CLIP/blob/HEAD/dataset/nr3d/add_evaluation_labels.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3c337a75a8d80a18"}},{"code_sha256_prefix":"6ed96e6cf33ac132","entry":"get_semantic_mapping_file","repo":"3dlg-hcvc/M3DRef-CLIP","repo_kind":"official","path":"dataset/scannetv2/preprocess_all_data.py","file_url":"https://github.com/3dlg-hcvc/M3DRef-CLIP/blob/HEAD/dataset/scannetv2/preprocess_all_data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6ed96e6cf33ac132"}},{"code_sha256_prefix":"bab2959c5abe5b6f","entry":"get_view_dependent_mask","repo":"3dlg-hcvc/M3DRef-CLIP","repo_kind":"official","path":"dataset/nr3d/add_evaluation_labels.py","file_url":"https://github.com/3dlg-hcvc/M3DRef-CLIP/blob/HEAD/dataset/nr3d/add_evaluation_labels.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bab2959c5abe5b6f"}},{"code_sha256_prefix":"5633e365874063be","entry":"read_axis_align_matrix","repo":"3dlg-hcvc/M3DRef-CLIP","repo_kind":"official","path":"dataset/scannetv2/preprocess_all_data.py","file_url":"https://github.com/3dlg-hcvc/M3DRef-CLIP/blob/HEAD/dataset/scannetv2/preprocess_all_data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5633e365874063be"}},{"code_sha256_prefix":"db79df35b7e4c4c2","entry":"generate_gt_multi3drefer","repo":"3dlg-hcvc/M3DRef-CLIP","repo_kind":"official","path":"evaluate.py","file_url":"https://github.com/3dlg-hcvc/M3DRef-CLIP/blob/HEAD/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"db79df35b7e4c4c2"}},{"code_sha256_prefix":"72af5b9cd6acffdd","entry":"generate_gt_nr3d","repo":"3dlg-hcvc/M3DRef-CLIP","repo_kind":"official","path":"evaluate.py","file_url":"https://github.com/3dlg-hcvc/M3DRef-CLIP/blob/HEAD/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"72af5b9cd6acffdd"}},{"code_sha256_prefix":"52d3233337b7e8cb","entry":"generate_gt_scanrefer","repo":"3dlg-hcvc/M3DRef-CLIP","repo_kind":"official","path":"evaluate.py","file_url":"https://github.com/3dlg-hcvc/M3DRef-CLIP/blob/HEAD/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"52d3233337b7e8cb"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}