{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/reason3d-searching-and-reasoning-3d","title":"Reason3D: Searching and Reasoning 3D Segmentation via Large Language Model","arxiv_id":"2405.17427","date":"2024-05-27","proceeding":null,"authors":["Kuan-Chih Huang","Xiangtai Li","Lu Qi","Shuicheng Yan","Ming-Hsuan Yang"],"abstract":"Recent advancements in multimodal large language models (LLMs) have shown their potential in various domains, especially concept reasoning. Despite these developments, applications in understanding 3D environments remain limited. This paper introduces Reason3D, a novel LLM designed for comprehensive 3D understanding. Reason3D takes point cloud data and text prompts as input to produce textual responses and segmentation masks, facilitating advanced tasks like 3D reasoning segmentation, hierarchical searching, express referring, and question answering with detailed mask outputs. Specifically, we propose a hierarchical mask decoder to locate small objects within expansive scenes. This decoder initially generates a coarse location estimate covering the object's general area. This foundational estimation facilitates a detailed, coarse-to-fine segmentation strategy that significantly enhances the precision of object identification and segmentation. Experiments validate that Reason3D achieves remarkable results on large-scale ScanNet and Matterport3D datasets for 3D express referring, 3D question answering, and 3D reasoning segmentation tasks. Code and models are available at: https://github.com/KuanchihHuang/Reason3D.","url_abs":"https://arxiv.org/abs/2405.17427v1","url_pdf":"https://arxiv.org/pdf/2405.17427v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"reason3d-searching-and-reasoning-3d","repo_url":"https://github.com/kuanchihhuang/reason3d","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"decoder","task_name":"Decoder"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"large-language-model","task_name":"Large Language Model"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"reasoning-segmentation","task_name":"Reasoning Segmentation"},{"task_slug":"segmentation","task_name":"Segmentation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2405.17427","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.17427"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kuanchihhuang/reason3d","reach":{"status":"ok"}}],"summary":{"ran":5,"unverified":2},"by_repo_kind":{"official":{"samples":7,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":7,"samples":[{"code_sha256_prefix":"911df8410fd2dad6","entry":"get_iou","repo":"kuanchihhuang/reason3d","repo_kind":"official","path":"lavis/models/reason3d_models/seg_loss.py","file_url":"https://github.com/kuanchihhuang/reason3d/blob/HEAD/lavis/models/reason3d_models/seg_loss.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"911df8410fd2dad6"}},{"code_sha256_prefix":"2f25075d0b84eb89","entry":"get_iou_prob","repo":"kuanchihhuang/reason3d","repo_kind":"official","path":"lavis/models/reason3d_models/seg_loss.py","file_url":"https://github.com/kuanchihhuang/reason3d/blob/HEAD/lavis/models/reason3d_models/seg_loss.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2f25075d0b84eb89"}},{"code_sha256_prefix":"791c72070c0b1cfb","entry":"node_to_dict","repo":"kuanchihhuang/reason3d","repo_kind":"official","path":"lavis/common/config.py","file_url":"https://github.com/kuanchihhuang/reason3d/blob/HEAD/lavis/common/config.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"791c72070c0b1cfb"}},{"code_sha256_prefix":"94629ea5eed6e387","entry":"sigmoid_focal_loss","repo":"kuanchihhuang/reason3d","repo_kind":"official","path":"lavis/models/reason3d_models/seg_loss.py","file_url":"https://github.com/kuanchihhuang/reason3d/blob/HEAD/lavis/models/reason3d_models/seg_loss.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"94629ea5eed6e387"}},{"code_sha256_prefix":"cb33571427334815","entry":"tile","repo":"kuanchihhuang/reason3d","repo_kind":"official","path":"lavis/models/base_model.py","file_url":"https://github.com/kuanchihhuang/reason3d/blob/HEAD/lavis/models/base_model.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"cb33571427334815"}},{"code_sha256_prefix":"0ec9fc2025c16f65","entry":"all_gather_with_grad","repo":"kuanchihhuang/reason3d","repo_kind":"official","path":"lavis/models/base_model.py","file_url":"https://github.com/kuanchihhuang/reason3d/blob/HEAD/lavis/models/base_model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0ec9fc2025c16f65"}},{"code_sha256_prefix":"bfbc7746415f3426","entry":"load_tf_weights_in_t5","repo":"kuanchihhuang/reason3d","repo_kind":"official","path":"lavis/models/reason3d_models/modeling_t5.py","file_url":"https://github.com/kuanchihhuang/reason3d/blob/HEAD/lavis/models/reason3d_models/modeling_t5.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bfbc7746415f3426"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}