{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/mono3dvg-3d-visual-grounding-in-monocular","title":"Mono3DVG: 3D Visual Grounding in Monocular Images","arxiv_id":"2312.08022","date":"2023-12-13","proceeding":null,"authors":["Yang Zhan","Yuan Yuan","Zhitong Xiong"],"abstract":"We introduce a novel task of 3D visual grounding in monocular RGB images using language descriptions with both appearance and geometry information. Specifically, we build a large-scale dataset, Mono3DRefer, which contains 3D object targets with their corresponding geometric text descriptions, generated by ChatGPT and refined manually. To foster this task, we propose Mono3DVG-TR, an end-to-end transformer-based network, which takes advantage of both the appearance and geometry information in text embeddings for multi-modal learning and 3D object localization. Depth predictor is designed to explicitly learn geometry features. The dual text-guided adapter is proposed to refine multiscale visual and geometry features of the referred object. Based on depth-text-visual stacking attention, the decoder fuses object-level geometric cues and visual appearance into a learnable query. Comprehensive benchmarks and some insightful analyses are provided for Mono3DVG. Extensive comparisons and ablation studies show that our method significantly outperforms all baselines. The dataset and code will be publicly available at: https://github.com/ZhanYang-nwpu/Mono3DVG.","url_abs":"https://arxiv.org/abs/2312.08022v1","url_pdf":"https://arxiv.org/pdf/2312.08022v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"mono3dvg-3d-visual-grounding-in-monocular","repo_url":"https://github.com/zhanyang-nwpu/mono3dvg","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"3d-object-detection","task_name":"3D Object Detection"},{"task_slug":"3d-visual-grounding","task_name":"3D visual grounding"},{"task_slug":"decoder","task_name":"Decoder"},{"task_slug":"mono3dvg","task_name":"Mono3DVG"},{"task_slug":"monocular-3d-object-detection","task_name":"Monocular 3D Object Detection"},{"task_slug":"object","task_name":"Object"},{"task_slug":"object-localization","task_name":"Object Localization"},{"task_slug":"visual-grounding","task_name":"Visual Grounding"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"adapter","method_name":"Adapter"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"deformable-attention-module","method_name":"Deformable Attention Module"},{"method_slug":"deformable-detr","method_name":"Deformable DETR"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"roberta","method_name":"RoBERTa"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[{"slug":"mono3drefer","name":"Mono3DRefer","full_name":""}],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2312.08022","atlas_url":"https://app.syntology.ai/?focus=2312.08022","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.08022"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/zhanyang-nwpu/mono3dvg","reach":{"status":"ok"}}],"summary":{"ran":4,"ran_draft_wrong":1,"unverified":2},"by_repo_kind":{"official":{"samples":7,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":7,"samples":[{"code_sha256_prefix":"39dc7b17b3bf3218","entry":"angle2class","repo":"zhanyang-nwpu/mono3dvg","repo_kind":"official","path":"lib/datasets/utils.py","file_url":"https://github.com/zhanyang-nwpu/mono3dvg/blob/HEAD/lib/datasets/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"39dc7b17b3bf3218"}},{"code_sha256_prefix":"b00e6da27bd2446e","entry":"build_optimizer","repo":"zhanyang-nwpu/mono3dvg","repo_kind":"official","path":"lib/helpers/optimizer_helper.py","file_url":"https://github.com/zhanyang-nwpu/mono3dvg/blob/HEAD/lib/helpers/optimizer_helper.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b00e6da27bd2446e"}},{"code_sha256_prefix":"fc35014925e8d9c7","entry":"class2angle","repo":"zhanyang-nwpu/mono3dvg","repo_kind":"official","path":"lib/datasets/utils.py","file_url":"https://github.com/zhanyang-nwpu/mono3dvg/blob/HEAD/lib/datasets/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fc35014925e8d9c7"}},{"code_sha256_prefix":"a1803f83e6407abc","entry":"get_checkpoint_state","repo":"zhanyang-nwpu/mono3dvg","repo_kind":"official","path":"lib/helpers/save_helper.py","file_url":"https://github.com/zhanyang-nwpu/mono3dvg/blob/HEAD/lib/helpers/save_helper.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a1803f83e6407abc"}},{"code_sha256_prefix":"51ce1f7d0c5be36c","entry":"model_state_to_cpu","repo":"zhanyang-nwpu/mono3dvg","repo_kind":"official","path":"lib/helpers/save_helper.py","file_url":"https://github.com/zhanyang-nwpu/mono3dvg/blob/HEAD/lib/helpers/save_helper.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"51ce1f7d0c5be36c"}},{"code_sha256_prefix":"d6f7bcb24f9c271a","entry":"gaussian_radius","repo":"zhanyang-nwpu/mono3dvg","repo_kind":"official","path":"lib/datasets/utils.py","file_url":"https://github.com/zhanyang-nwpu/mono3dvg/blob/HEAD/lib/datasets/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d6f7bcb24f9c271a"}},{"code_sha256_prefix":"44a0a6740ca07a2b","entry":"load_checkpoint","repo":"zhanyang-nwpu/mono3dvg","repo_kind":"official","path":"lib/helpers/save_helper.py","file_url":"https://github.com/zhanyang-nwpu/mono3dvg/blob/HEAD/lib/helpers/save_helper.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"44a0a6740ca07a2b"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}