{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/llm-grounder-open-vocabulary-3d-visual","title":"LLM-Grounder: Open-Vocabulary 3D Visual Grounding with Large Language Model as an Agent","arxiv_id":"2309.12311","date":"2023-09-21","proceeding":null,"authors":["Jianing Yang","Xuweiyi Chen","Shengyi Qian","Nikhil Madaan","Madhavan Iyengar","David F. Fouhey","Joyce Chai"],"abstract":"3D visual grounding is a critical skill for household robots, enabling them to navigate, manipulate objects, and answer questions based on their environment. While existing approaches often rely on extensive labeled data or exhibit limitations in handling complex language queries, we propose LLM-Grounder, a novel zero-shot, open-vocabulary, Large Language Model (LLM)-based 3D visual grounding pipeline. LLM-Grounder utilizes an LLM to decompose complex natural language queries into semantic constituents and employs a visual grounding tool, such as OpenScene or LERF, to identify objects in a 3D scene. The LLM then evaluates the spatial and commonsense relations among the proposed objects to make a final grounding decision. Our method does not require any labeled training data and can generalize to novel 3D scenes and arbitrary text queries. We evaluate LLM-Grounder on the ScanRefer benchmark and demonstrate state-of-the-art zero-shot grounding accuracy. Our findings indicate that LLMs significantly improve the grounding capability, especially for complex language queries, making LLM-Grounder an effective approach for 3D vision-language tasks in robotics. Videos and interactive demos can be found on the project website https://chat-with-nerf.github.io/ .","url_abs":"https://arxiv.org/abs/2309.12311v1","url_pdf":"https://arxiv.org/pdf/2309.12311v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"llm-grounder-open-vocabulary-3d-visual","repo_url":"https://github.com/sled-group/chat-with-nerf","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"3d-visual-grounding","task_name":"3D visual grounding"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"large-language-model","task_name":"Large Language Model"},{"task_slug":"natural-language-queries","task_name":"Natural Language Queries"},{"task_slug":"navigate","task_name":"Navigate"},{"task_slug":"visual-grounding","task_name":"Visual Grounding"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2309.12311","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2309.12311"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sled-group/chat-with-nerf","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":5},"by_repo_kind":{"official":{"samples":5,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"c26cdea93a23aef7","entry":"fix_brackets","repo":"sled-group/chat-with-nerf","repo_kind":"official","path":"chat_with_nerf/chat/util.py","file_url":"https://github.com/sled-group/chat-with-nerf/blob/HEAD/chat_with_nerf/chat/util.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c26cdea93a23aef7"}},{"code_sha256_prefix":"680b7652118076c7","entry":"list_dirs","repo":"sled-group/chat-with-nerf","repo_kind":"official","path":"chat_with_nerf/util.py","file_url":"https://github.com/sled-group/chat-with-nerf/blob/HEAD/chat_with_nerf/util.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"680b7652118076c7"}},{"code_sha256_prefix":"d84bc4d543f3686f","entry":"robust_json_loads","repo":"sled-group/chat-with-nerf","repo_kind":"official","path":"chat_with_nerf/chat/util.py","file_url":"https://github.com/sled-group/chat-with-nerf/blob/HEAD/chat_with_nerf/chat/util.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d84bc4d543f3686f"}},{"code_sha256_prefix":"583921d3809ca52e","entry":"rotate_x","repo":"sled-group/chat-with-nerf","repo_kind":"official","path":"chat_with_nerf/util.py","file_url":"https://github.com/sled-group/chat-with-nerf/blob/HEAD/chat_with_nerf/util.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"583921d3809ca52e"}},{"code_sha256_prefix":"d53f6a6a7ec67408","entry":"rotate_y","repo":"sled-group/chat-with-nerf","repo_kind":"official","path":"chat_with_nerf/util.py","file_url":"https://github.com/sled-group/chat-with-nerf/blob/HEAD/chat_with_nerf/util.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d53f6a6a7ec67408"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}