{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/multi-modal-fusion-transformer-for-end-to-end","title":"Multi-Modal Fusion Transformer for End-to-End Autonomous Driving","arxiv_id":"2104.09224","date":"2021-04-19","proceeding":"CVPR 2021 1","authors":["Aditya Prakash","Kashyap Chitta","Andreas Geiger"],"abstract":"How should representations from complementary sensors be integrated for autonomous driving? Geometry-based sensor fusion has shown great promise for perception tasks such as object detection and motion forecasting. However, for the actual driving task, the global context of the 3D scene is key, e.g. a change in traffic light state can affect the behavior of a vehicle geometrically distant from that traffic light. Geometry alone may therefore be insufficient for effectively fusing representations in end-to-end driving models. In this work, we demonstrate that imitation learning policies based on existing sensor fusion methods under-perform in the presence of a high density of dynamic agents and complex scenarios, which require global contextual reasoning, such as handling traffic oncoming from multiple directions at uncontrolled intersections. Therefore, we propose TransFuser, a novel Multi-Modal Fusion Transformer, to integrate image and LiDAR representations using attention. We experimentally validate the efficacy of our approach in urban settings involving complex scenarios using the CARLA urban driving simulator. Our approach achieves state-of-the-art driving performance while reducing collisions by 76% compared to geometry-based fusion.","url_abs":"https://arxiv.org/abs/2104.09224v1","url_pdf":"https://arxiv.org/pdf/2104.09224v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"multi-modal-fusion-transformer-for-end-to-end","repo_url":"https://github.com/autonomousvision/transfuser","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"multi-modal-fusion-transformer-for-end-to-end","repo_url":"https://github.com/Kin-Zhang/mmfn","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"autonomous-driving","task_name":"Autonomous Driving"},{"task_slug":"imitation-learning","task_name":"Imitation Learning"},{"task_slug":"motion-forecasting","task_name":"Motion Forecasting"},{"task_slug":"object-detection","task_name":"Object Detection"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"},{"task_slug":"sensor-fusion","task_name":"Sensor Fusion"},{"task_slug":"object-detection-1","task_name":"object-detection"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"carla","method_name":"CARLA"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"entropy-regularization","method_name":"Entropy Regularization"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"ppo","method_name":"PPO"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/autonomous-driving-on-carla-leaderboard","task":"Autonomous Driving","dataset":"CARLA Leaderboard","model":"Transfuser","rank_in_archive_order":16,"of":18,"metrics":{"Driving Score":"16.93","Infraction penalty":"0.42","Route Completion":"51.82"},"uses_additional_data":false},{"leaderboard":"/sota/autonomous-driving-on-town05-long","task":"Autonomous Driving","dataset":"Town05 Long","model":"Geometric Fusion","rank_in_archive_order":1,"of":2,"metrics":{"RC":"69.17"},"uses_additional_data":false},{"leaderboard":"/sota/autonomous-driving-on-town05-long","task":"Autonomous Driving","dataset":"Town05 Long","model":"TransFuser","rank_in_archive_order":2,"of":2,"metrics":{"DS":"33.15","RC":"56.36"},"uses_additional_data":false},{"leaderboard":"/sota/autonomous-driving-on-town05-short","task":"Autonomous Driving","dataset":"Town05 Short","model":"Geometric Fusion","rank_in_archive_order":1,"of":2,"metrics":{"RC":"86.91"},"uses_additional_data":false},{"leaderboard":"/sota/autonomous-driving-on-town05-short","task":"Autonomous Driving","dataset":"Town05 Short","model":"TransFuser","rank_in_archive_order":2,"of":2,"metrics":{"DS":"54.52","RC":"78.41"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-segmentation-on-kitti-360","task":"Semantic Segmentation","dataset":"KITTI-360","model":"TransFuser (RGB-LiDAR)","rank_in_archive_order":10,"of":17,"metrics":{"mIoU":"56.57"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2104.09224","atlas_url":"https://app.syntology.ai/?focus=2104.09224","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.09224"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/autonomousvision/transfuser","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Kin-Zhang/mmfn","reach":null}],"summary":{"ran":7,"ran_violates":1,"ran_draft_wrong":1,"unverified":6},"by_repo_kind":{"official":{"samples":6,"ran":3,"repositories":1},"listed":{"samples":9,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"5075118d0c207c37","entry":"Block","repo":"Kin-Zhang/mmfn","repo_kind":"listed","path":"team_code/benchmarks/transfuser/model.py","file_url":"https://github.com/Kin-Zhang/mmfn/blob/HEAD/team_code/benchmarks/transfuser/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5075118d0c207c37"}},{"code_sha256_prefix":"d93ea624e09cfe3d","entry":"Block","repo":"autonomousvision/transfuser","repo_kind":"official","path":"team_code_transfuser/transfuser.py","file_url":"https://github.com/autonomousvision/transfuser/blob/HEAD/team_code_transfuser/transfuser.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d93ea624e09cfe3d"}},{"code_sha256_prefix":"594894f4b9eb35a1","entry":"GPT","repo":"Kin-Zhang/mmfn","repo_kind":"listed","path":"team_code/benchmarks/transfuser/model.py","file_url":"https://github.com/Kin-Zhang/mmfn/blob/HEAD/team_code/benchmarks/transfuser/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"594894f4b9eb35a1"}},{"code_sha256_prefix":"7a6582011815aa73","entry":"GPT","repo":"autonomousvision/transfuser","repo_kind":"official","path":"team_code_transfuser/transfuser.py","file_url":"https://github.com/autonomousvision/transfuser/blob/HEAD/team_code_transfuser/transfuser.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7a6582011815aa73"}},{"code_sha256_prefix":"152cd035838391cd","entry":"LidarEncoder","repo":"Kin-Zhang/mmfn","repo_kind":"listed","path":"team_code/benchmarks/transfuser/model.py","file_url":"https://github.com/Kin-Zhang/mmfn/blob/HEAD/team_code/benchmarks/transfuser/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"152cd035838391cd"}},{"code_sha256_prefix":"bd73b9145f2b9944","entry":"PIDController","repo":"Kin-Zhang/mmfn","repo_kind":"listed","path":"team_code/benchmarks/transfuser/model.py","file_url":"https://github.com/Kin-Zhang/mmfn/blob/HEAD/team_code/benchmarks/transfuser/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bd73b9145f2b9944"}},{"code_sha256_prefix":"6141dbd1107c57fd","entry":"SelfAttention","repo":"Kin-Zhang/mmfn","repo_kind":"listed","path":"team_code/benchmarks/transfuser/model.py","file_url":"https://github.com/Kin-Zhang/mmfn/blob/HEAD/team_code/benchmarks/transfuser/model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6141dbd1107c57fd"}},{"code_sha256_prefix":"af36a314f4f4a4be","entry":"normalize_imagenet","repo":"Kin-Zhang/mmfn","repo_kind":"listed","path":"team_code/benchmarks/transfuser/model.py","file_url":"https://github.com/Kin-Zhang/mmfn/blob/HEAD/team_code/benchmarks/transfuser/model.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"af36a314f4f4a4be"}},{"code_sha256_prefix":"1529467107cafc96","entry":"normalize_imagenet","repo":"autonomousvision/transfuser","repo_kind":"official","path":"team_code_transfuser/transfuser.py","file_url":"https://github.com/autonomousvision/transfuser/blob/HEAD/team_code_transfuser/transfuser.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1529467107cafc96"}},{"code_sha256_prefix":"277c052f581f3276","entry":"Encoder","repo":"Kin-Zhang/mmfn","repo_kind":"listed","path":"team_code/benchmarks/transfuser/model.py","file_url":"https://github.com/Kin-Zhang/mmfn/blob/HEAD/team_code/benchmarks/transfuser/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"277c052f581f3276"}},{"code_sha256_prefix":"1a4dc7d1e43090ab","entry":"ImageCNN","repo":"Kin-Zhang/mmfn","repo_kind":"listed","path":"team_code/benchmarks/transfuser/model.py","file_url":"https://github.com/Kin-Zhang/mmfn/blob/HEAD/team_code/benchmarks/transfuser/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1a4dc7d1e43090ab"}},{"code_sha256_prefix":"e1a45fd31eaf668d","entry":"ImageCNN","repo":"autonomousvision/transfuser","repo_kind":"official","path":"team_code_transfuser/transfuser.py","file_url":"https://github.com/autonomousvision/transfuser/blob/HEAD/team_code_transfuser/transfuser.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e1a45fd31eaf668d"}},{"code_sha256_prefix":"d2d93b6e784851f4","entry":"LidarEncoder","repo":"autonomousvision/transfuser","repo_kind":"official","path":"team_code_transfuser/transfuser.py","file_url":"https://github.com/autonomousvision/transfuser/blob/HEAD/team_code_transfuser/transfuser.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d2d93b6e784851f4"}},{"code_sha256_prefix":"3d3c28ac7ec98753","entry":"TransFuser","repo":"Kin-Zhang/mmfn","repo_kind":"listed","path":"team_code/benchmarks/transfuser/model.py","file_url":"https://github.com/Kin-Zhang/mmfn/blob/HEAD/team_code/benchmarks/transfuser/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3d3c28ac7ec98753"}},{"code_sha256_prefix":"8d4dbe093612822d","entry":"TransfuserBackbone","repo":"autonomousvision/transfuser","repo_kind":"official","path":"team_code_transfuser/transfuser.py","file_url":"https://github.com/autonomousvision/transfuser/blob/HEAD/team_code_transfuser/transfuser.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8d4dbe093612822d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}