{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/futr3d-a-unified-sensor-fusion-framework-for","title":"FUTR3D: A Unified Sensor Fusion Framework for 3D Detection","arxiv_id":"2203.10642","date":"2022-03-20","proceeding":null,"authors":["Xuanyao Chen","Tianyuan Zhang","Yue Wang","Yilun Wang","Hang Zhao"],"abstract":"Sensor fusion is an essential topic in many perception systems, such as autonomous driving and robotics. Existing multi-modal 3D detection models usually involve customized designs depending on the sensor combinations or setups. In this work, we propose the first unified end-to-end sensor fusion framework for 3D detection, named FUTR3D, which can be used in (almost) any sensor configuration. FUTR3D employs a query-based Modality-Agnostic Feature Sampler (MAFS), together with a transformer decoder with a set-to-set loss for 3D detection, thus avoiding using late fusion heuristics and post-processing tricks. We validate the effectiveness of our framework on various combinations of cameras, low-resolution LiDARs, high-resolution LiDARs, and Radars. On NuScenes dataset, FUTR3D achieves better performance over specifically designed methods across different sensor combinations. Moreover, FUTR3D achieves great flexibility with different sensor configurations and enables low-cost autonomous driving. For example, only using a 4-beam LiDAR with cameras, FUTR3D (58.0 mAP) achieves on par performance with state-of-the-art 3D detection model CenterPoint (56.6 mAP) using a 32-beam LiDAR.","url_abs":"https://arxiv.org/abs/2203.10642v2","url_pdf":"https://arxiv.org/pdf/2203.10642v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"futr3d-a-unified-sensor-fusion-framework-for","repo_url":"https://github.com/tsinghua-mars-lab/futr3d","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"futr3d-a-unified-sensor-fusion-framework-for","repo_url":"https://github.com/xiaohao-xu/unified-pretrain-ad","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"autonomous-driving","task_name":"Autonomous Driving"},{"task_slug":"decoder","task_name":"Decoder"},{"task_slug":"sensor-fusion","task_name":"Sensor Fusion"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2203.10642","atlas_url":"https://app.syntology.ai/?focus=2203.10642","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2203.10642"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/tsinghua-mars-lab/futr3d","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/xiaohao-xu/unified-pretrain-ad","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":9,"unverified":10},"by_repo_kind":{"listed":{"samples":19,"ran":9,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"c637c42a302bb63c","entry":"conv1x1","repo":"tsinghua-mars-lab/futr3d","repo_kind":"listed","path":"plugin/futr3d/models/backbone/vovnet.py","file_url":"https://github.com/tsinghua-mars-lab/futr3d/blob/HEAD/plugin/futr3d/models/backbone/vovnet.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c637c42a302bb63c"}},{"code_sha256_prefix":"303f4d3695ad18f1","entry":"conv3x3","repo":"tsinghua-mars-lab/futr3d","repo_kind":"listed","path":"plugin/futr3d/models/backbone/vovnet.py","file_url":"https://github.com/tsinghua-mars-lab/futr3d/blob/HEAD/plugin/futr3d/models/backbone/vovnet.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"303f4d3695ad18f1"}},{"code_sha256_prefix":"b8cdf4cad83ce917","entry":"denormalize_bbox","repo":"tsinghua-mars-lab/futr3d","repo_kind":"listed","path":"plugin/futr3d/core/bbox/utils.py","file_url":"https://github.com/tsinghua-mars-lab/futr3d/blob/HEAD/plugin/futr3d/core/bbox/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b8cdf4cad83ce917"}},{"code_sha256_prefix":"38e90d12ef38fe79","entry":"denormalize_bbox","repo":"xiaohao-xu/unified-pretrain-ad","repo_kind":"listed","path":"mmdet3d/core/bbox/util.py","file_url":"https://github.com/xiaohao-xu/unified-pretrain-ad/blob/HEAD/mmdet3d/core/bbox/util.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"38e90d12ef38fe79"}},{"code_sha256_prefix":"ac73da5ed8412961","entry":"feature_sampling","repo":"tsinghua-mars-lab/futr3d","repo_kind":"listed","path":"plugin/futr3d/models/utils/futr3d_attention.py","file_url":"https://github.com/tsinghua-mars-lab/futr3d/blob/HEAD/plugin/futr3d/models/utils/futr3d_attention.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ac73da5ed8412961"}},{"code_sha256_prefix":"ea036153b550642a","entry":"gaussian_2d","repo":"xiaohao-xu/unified-pretrain-ad","repo_kind":"listed","path":"mmdet3d/core/utils/gaussian.py","file_url":"https://github.com/xiaohao-xu/unified-pretrain-ad/blob/HEAD/mmdet3d/core/utils/gaussian.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ea036153b550642a"}},{"code_sha256_prefix":"5e13611ee307d69a","entry":"get_paddings_indicator","repo":"tsinghua-mars-lab/futr3d","repo_kind":"listed","path":"plugin/futr3d/models/backbone/radar_encoder.py","file_url":"https://github.com/tsinghua-mars-lab/futr3d/blob/HEAD/plugin/futr3d/models/backbone/radar_encoder.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5e13611ee307d69a"}},{"code_sha256_prefix":"8800486c70b30b88","entry":"normalize_bbox","repo":"tsinghua-mars-lab/futr3d","repo_kind":"listed","path":"plugin/futr3d/core/bbox/utils.py","file_url":"https://github.com/tsinghua-mars-lab/futr3d/blob/HEAD/plugin/futr3d/core/bbox/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8800486c70b30b88"}},{"code_sha256_prefix":"e6ab81c2ff932920","entry":"normalize_bbox","repo":"xiaohao-xu/unified-pretrain-ad","repo_kind":"listed","path":"mmdet3d/core/bbox/util.py","file_url":"https://github.com/xiaohao-xu/unified-pretrain-ad/blob/HEAD/mmdet3d/core/bbox/util.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e6ab81c2ff932920"}},{"code_sha256_prefix":"0079ed3e99e4048d","entry":"aligned_3d_nms","repo":"xiaohao-xu/unified-pretrain-ad","repo_kind":"listed","path":"mmdet3d/core/post_processing/box3d_nms.py","file_url":"https://github.com/xiaohao-xu/unified-pretrain-ad/blob/HEAD/mmdet3d/core/post_processing/box3d_nms.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0079ed3e99e4048d"}},{"code_sha256_prefix":"96c8084488bd023b","entry":"box_camera_to_lidar","repo":"xiaohao-xu/unified-pretrain-ad","repo_kind":"listed","path":"mmdet3d/core/bbox/box_np_ops.py","file_url":"https://github.com/xiaohao-xu/unified-pretrain-ad/blob/HEAD/mmdet3d/core/bbox/box_np_ops.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"96c8084488bd023b"}},{"code_sha256_prefix":"9e22984820c39868","entry":"camera_to_lidar","repo":"xiaohao-xu/unified-pretrain-ad","repo_kind":"listed","path":"mmdet3d/core/bbox/box_np_ops.py","file_url":"https://github.com/xiaohao-xu/unified-pretrain-ad/blob/HEAD/mmdet3d/core/bbox/box_np_ops.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9e22984820c39868"}},{"code_sha256_prefix":"a8007c7a213459b2","entry":"circle_nms","repo":"xiaohao-xu/unified-pretrain-ad","repo_kind":"listed","path":"mmdet3d/core/post_processing/box3d_nms.py","file_url":"https://github.com/xiaohao-xu/unified-pretrain-ad/blob/HEAD/mmdet3d/core/post_processing/box3d_nms.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a8007c7a213459b2"}},{"code_sha256_prefix":"5b1dacd1f125c08f","entry":"corners_nd","repo":"xiaohao-xu/unified-pretrain-ad","repo_kind":"listed","path":"mmdet3d/core/bbox/box_np_ops.py","file_url":"https://github.com/xiaohao-xu/unified-pretrain-ad/blob/HEAD/mmdet3d/core/bbox/box_np_ops.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5b1dacd1f125c08f"}},{"code_sha256_prefix":"a7cde40ccc6422e2","entry":"draw_heatmap_gaussian","repo":"xiaohao-xu/unified-pretrain-ad","repo_kind":"listed","path":"mmdet3d/core/utils/gaussian.py","file_url":"https://github.com/xiaohao-xu/unified-pretrain-ad/blob/HEAD/mmdet3d/core/utils/gaussian.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a7cde40ccc6422e2"}},{"code_sha256_prefix":"5ebf15e6940f6710","entry":"dw_conv3x3","repo":"tsinghua-mars-lab/futr3d","repo_kind":"listed","path":"plugin/futr3d/models/backbone/vovnet.py","file_url":"https://github.com/tsinghua-mars-lab/futr3d/blob/HEAD/plugin/futr3d/models/backbone/vovnet.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5ebf15e6940f6710"}},{"code_sha256_prefix":"73e8316be86a800d","entry":"gaussian_radius","repo":"xiaohao-xu/unified-pretrain-ad","repo_kind":"listed","path":"mmdet3d/core/utils/gaussian.py","file_url":"https://github.com/xiaohao-xu/unified-pretrain-ad/blob/HEAD/mmdet3d/core/utils/gaussian.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"73e8316be86a800d"}},{"code_sha256_prefix":"f3855be53d36327d","entry":"inverse_sigmoid","repo":"tsinghua-mars-lab/futr3d","repo_kind":"listed","path":"plugin/futr3d/models/utils/futr3d_attention.py","file_url":"https://github.com/tsinghua-mars-lab/futr3d/blob/HEAD/plugin/futr3d/models/utils/futr3d_attention.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f3855be53d36327d"}},{"code_sha256_prefix":"d061ed2b287db1a4","entry":"reduce_LiDAR_beams","repo":"tsinghua-mars-lab/futr3d","repo_kind":"listed","path":"plugin/futr3d/datasets/loading.py","file_url":"https://github.com/tsinghua-mars-lab/futr3d/blob/HEAD/plugin/futr3d/datasets/loading.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d061ed2b287db1a4"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}