{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/refer-it-in-rgbd-a-bottom-up-approach-for-3d","title":"Refer-it-in-RGBD: A Bottom-up Approach for 3D Visual Grounding in RGBD Images","arxiv_id":"2103.07894","date":"2021-03-14","proceeding":"CVPR 2021 1","authors":["Haolin Liu","Anran Lin","Xiaoguang Han","Lei Yang","Yizhou Yu","Shuguang Cui"],"abstract":"Grounding referring expressions in RGBD image has been an emerging field. We present a novel task of 3D visual grounding in single-view RGBD image where the referred objects are often only partially scanned due to occlusion. In contrast to previous works that directly generate object proposals for grounding in the 3D scenes, we propose a bottom-up approach to gradually aggregate context-aware information, effectively addressing the challenge posed by the partial geometry. Our approach first fuses the language and the visual features at the bottom level to generate a heatmap that coarsely localizes the relevant regions in the RGBD image. Then our approach conducts an adaptive feature learning based on the heatmap and performs the object-level matching with another visio-linguistic fusion to finally ground the referred object. We evaluate the proposed method by comparing to the state-of-the-art methods on both the RGBD images extracted from the ScanRefer dataset and our newly collected SUNRefer dataset. Experiments show that our method outperforms the previous methods by a large margin (by 11.2% and 15.6% Acc@0.5) on both datasets.","url_abs":"https://arxiv.org/abs/2103.07894v3","url_pdf":"https://arxiv.org/pdf/2103.07894v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"refer-it-in-rgbd-a-bottom-up-approach-for-3d","repo_url":"https://github.com/UncleMEDM/Refer-it-in-RGBD","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"refer-it-in-rgbd-a-bottom-up-approach-for-3d","repo_url":"https://github.com/unclemedm/instpifu","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"3d-visual-grounding","task_name":"3D visual grounding"},{"task_slug":"object","task_name":"Object"},{"task_slug":"visual-grounding","task_name":"Visual Grounding"}],"methods":[{"method_slug":"heatmap","method_name":"Heatmap"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2103.07894","atlas_url":"https://app.syntology.ai/?focus=2103.07894","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2103.07894"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/UncleMEDM/Refer-it-in-RGBD","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/unclemedm/instpifu","reach":null}],"summary":{"ran":10,"ran_draft_wrong":1,"unverified":5},"by_repo_kind":{"listed":{"samples":16,"ran":11,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":6,"samples":[{"code_sha256_prefix":"ef23e429bf663c1f","entry":"BallQuery","repo":"UncleMEDM/Refer-it-in-RGBD","repo_kind":"listed","path":"network/ref_net.py","file_url":"https://github.com/UncleMEDM/Refer-it-in-RGBD/blob/HEAD/network/ref_net.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ef23e429bf663c1f"}},{"code_sha256_prefix":"732128aba96a1d0c","entry":"BasicBlock","repo":"unclemedm/instpifu","repo_kind":"listed","path":"models/detection/object_detection.py","file_url":"https://github.com/unclemedm/instpifu/blob/HEAD/models/detection/object_detection.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"732128aba96a1d0c"}},{"code_sha256_prefix":"e52fe679d8feec16","entry":"FurthestPointSampling","repo":"UncleMEDM/Refer-it-in-RGBD","repo_kind":"listed","path":"network/ref_net.py","file_url":"https://github.com/UncleMEDM/Refer-it-in-RGBD/blob/HEAD/network/ref_net.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e52fe679d8feec16"}},{"code_sha256_prefix":"74f000897bda48e2","entry":"GatherOperation","repo":"UncleMEDM/Refer-it-in-RGBD","repo_kind":"listed","path":"network/ref_net.py","file_url":"https://github.com/UncleMEDM/Refer-it-in-RGBD/blob/HEAD/network/ref_net.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"74f000897bda48e2"}},{"code_sha256_prefix":"c2ed988623fd564a","entry":"GroupAll","repo":"UncleMEDM/Refer-it-in-RGBD","repo_kind":"listed","path":"network/ref_net.py","file_url":"https://github.com/UncleMEDM/Refer-it-in-RGBD/blob/HEAD/network/ref_net.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c2ed988623fd564a"}},{"code_sha256_prefix":"3d1e2708286ed3ad","entry":"GroupingOperation","repo":"UncleMEDM/Refer-it-in-RGBD","repo_kind":"listed","path":"network/ref_net.py","file_url":"https://github.com/UncleMEDM/Refer-it-in-RGBD/blob/HEAD/network/ref_net.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3d1e2708286ed3ad"}},{"code_sha256_prefix":"54036bdab4e46cd3","entry":"QueryAndGroup","repo":"UncleMEDM/Refer-it-in-RGBD","repo_kind":"listed","path":"network/ref_net.py","file_url":"https://github.com/UncleMEDM/Refer-it-in-RGBD/blob/HEAD/network/ref_net.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"54036bdab4e46cd3"}},{"code_sha256_prefix":"4abd4c9066a90156","entry":"Relation_Config","repo":"unclemedm/instpifu","repo_kind":"listed","path":"models/detection/object_detection.py","file_url":"https://github.com/unclemedm/instpifu/blob/HEAD/models/detection/object_detection.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4abd4c9066a90156"}},{"code_sha256_prefix":"d3509abedccb0f1f","entry":"ResNet","repo":"unclemedm/instpifu","repo_kind":"listed","path":"models/detection/object_detection.py","file_url":"https://github.com/unclemedm/instpifu/blob/HEAD/models/detection/object_detection.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d3509abedccb0f1f"}},{"code_sha256_prefix":"1004f78d4c844560","entry":"Weighted_FurthestPointSampling","repo":"UncleMEDM/Refer-it-in-RGBD","repo_kind":"listed","path":"network/ref_net.py","file_url":"https://github.com/UncleMEDM/Refer-it-in-RGBD/blob/HEAD/network/ref_net.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1004f78d4c844560"}},{"code_sha256_prefix":"9cd7bf8cbc6b7671","entry":"resnet34","repo":"unclemedm/instpifu","repo_kind":"listed","path":"models/detection/object_detection.py","file_url":"https://github.com/unclemedm/instpifu/blob/HEAD/models/detection/object_detection.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9cd7bf8cbc6b7671"}},{"code_sha256_prefix":"fb43534ffd4e20c9","entry":"Bdb3DNet","repo":"unclemedm/instpifu","repo_kind":"listed","path":"models/detection/object_detection.py","file_url":"https://github.com/unclemedm/instpifu/blob/HEAD/models/detection/object_detection.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fb43534ffd4e20c9"}},{"code_sha256_prefix":"7677e3ee9ff8db40","entry":"PointnetLFPModuleMSG","repo":"UncleMEDM/Refer-it-in-RGBD","repo_kind":"listed","path":"network/ref_net.py","file_url":"https://github.com/UncleMEDM/Refer-it-in-RGBD/blob/HEAD/network/ref_net.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7677e3ee9ff8db40"}},{"code_sha256_prefix":"9bb0104cc116ccb9","entry":"PointnetSAModuleVotes","repo":"UncleMEDM/Refer-it-in-RGBD","repo_kind":"listed","path":"network/ref_net.py","file_url":"https://github.com/UncleMEDM/Refer-it-in-RGBD/blob/HEAD/network/ref_net.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9bb0104cc116ccb9"}},{"code_sha256_prefix":"300df34b1ec73570","entry":"RGBD_RefNet","repo":"UncleMEDM/Refer-it-in-RGBD","repo_kind":"listed","path":"network/ref_net.py","file_url":"https://github.com/UncleMEDM/Refer-it-in-RGBD/blob/HEAD/network/ref_net.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"300df34b1ec73570"}},{"code_sha256_prefix":"aa227bfc5b89714a","entry":"RelationNet","repo":"unclemedm/instpifu","repo_kind":"listed","path":"models/detection/object_detection.py","file_url":"https://github.com/unclemedm/instpifu/blob/HEAD/models/detection/object_detection.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"aa227bfc5b89714a"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}