{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/transfuser-imitation-with-transformer-based","title":"TransFuser: Imitation with Transformer-Based Sensor Fusion for Autonomous Driving","arxiv_id":"2205.15997","date":"2022-05-31","proceeding":null,"authors":["Kashyap Chitta","Aditya Prakash","Bernhard Jaeger","Zehao Yu","Katrin Renz","Andreas Geiger"],"abstract":"How should we integrate representations from complementary sensors for autonomous driving? Geometry-based fusion has shown promise for perception (e.g. object detection, motion forecasting). However, in the context of end-to-end driving, we find that imitation learning based on existing sensor fusion methods underperforms in complex driving scenarios with a high density of dynamic agents. Therefore, we propose TransFuser, a mechanism to integrate image and LiDAR representations using self-attention. Our approach uses transformer modules at multiple resolutions to fuse perspective view and bird's eye view feature maps. We experimentally validate its efficacy on a challenging new benchmark with long routes and dense traffic, as well as the official leaderboard of the CARLA urban driving simulator. At the time of submission, TransFuser outperforms all prior work on the CARLA leaderboard in terms of driving score by a large margin. Compared to geometry-based fusion, TransFuser reduces the average collisions per kilometer by 48%.","url_abs":"https://arxiv.org/abs/2205.15997v1","url_pdf":"https://arxiv.org/pdf/2205.15997v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"transfuser-imitation-with-transformer-based","repo_url":"https://github.com/autonomousvision/transfuser","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"transfuser-imitation-with-transformer-based","repo_url":"https://github.com/autonomousvision/navsim","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":null},{"paper_slug":"transfuser-imitation-with-transformer-based","repo_url":"https://github.com/pagand/e2etransfuser","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"autonomous-driving","task_name":"Autonomous Driving"},{"task_slug":"carla-longest6","task_name":"CARLA longest6"},{"task_slug":"imitation-learning","task_name":"Imitation Learning"},{"task_slug":"motion-forecasting","task_name":"Motion Forecasting"},{"task_slug":"navsim","task_name":"NavSim"},{"task_slug":"object-detection","task_name":"Object Detection"},{"task_slug":"sensor-fusion","task_name":"Sensor Fusion"}],"methods":[{"method_slug":"carla","method_name":"CARLA"},{"method_slug":"entropy-regularization","method_name":"Entropy Regularization"},{"method_slug":"ppo","method_name":"PPO"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/autonomous-driving-on-carla-leaderboard","task":"Autonomous Driving","dataset":"CARLA Leaderboard","model":"TransFuser","rank_in_archive_order":6,"of":18,"metrics":{"Driving Score":"61.181","Infraction penalty":"0.714","Route Completion":"86.694"},"uses_additional_data":false},{"leaderboard":"/sota/autonomous-driving-on-carla-leaderboard","task":"Autonomous Driving","dataset":"CARLA Leaderboard","model":"TransFuser (Reproduced)","rank_in_archive_order":7,"of":18,"metrics":{"Driving Score":"55.04","Infraction penalty":"0.63","Route Completion":"89.65"},"uses_additional_data":false},{"leaderboard":"/sota/autonomous-driving-on-carla-leaderboard","task":"Autonomous Driving","dataset":"CARLA Leaderboard","model":"Latent TransFuser","rank_in_archive_order":9,"of":18,"metrics":{"Driving Score":"45.20","Infraction penalty":"0.72","Route Completion":"66.31"},"uses_additional_data":false},{"leaderboard":"/sota/carla-longest6-on-carla","task":"CARLA longest6","dataset":"CARLA","model":"TransFuser (TF)","rank_in_archive_order":14,"of":21,"metrics":{"Driving Score":"47","Infraction Score":"0.50","Route Completion":"93"},"uses_additional_data":false},{"leaderboard":"/sota/carla-longest6-on-carla","task":"CARLA longest6","dataset":"CARLA","model":"Latent TransFuser (LTF)","rank_in_archive_order":16,"of":21,"metrics":{"Driving Score":"37","Infraction Score":"0.38","Route Completion":"95"},"uses_additional_data":false},{"leaderboard":"/sota/carla-longest6-on-carla","task":"CARLA longest6","dataset":"CARLA","model":"Geometric Fusion (GF)","rank_in_archive_order":18,"of":21,"metrics":{"Driving Score":"27","Infraction Score":"0.30","Route Completion":"91"},"uses_additional_data":false},{"leaderboard":"/sota/carla-longest6-on-carla","task":"CARLA longest6","dataset":"CARLA","model":"Late Fusion (LF)","rank_in_archive_order":20,"of":21,"metrics":{"Driving Score":"22","Infraction Score":"0.27","Route Completion":"83"},"uses_additional_data":false},{"leaderboard":"/sota/navsim-on-openscene","task":"NavSim","dataset":"OpenScene","model":"TransFuser","rank_in_archive_order":20,"of":29,"metrics":{"PDMS":"84.0"},"uses_additional_data":false},{"leaderboard":"/sota/navsim-on-openscene","task":"NavSim","dataset":"OpenScene","model":"Latent TransFuser","rank_in_archive_order":22,"of":29,"metrics":{"PDMS":"83.8"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2205.15997","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2205.15997"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/autonomousvision/navsim","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/autonomousvision/transfuser","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/pagand/e2etransfuser","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_violates":1,"unverified":4},"by_repo_kind":{"listed":{"samples":5,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"af36a314f4f4a4be","entry":"normalize_imagenet","repo":"pagand/e2etransfuser","repo_kind":"listed","path":"aim/model.py","file_url":"https://github.com/pagand/e2etransfuser/blob/HEAD/aim/model.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"af36a314f4f4a4be"}},{"code_sha256_prefix":"6f4eddf7f8def829","entry":"IOU","repo":"pagand/e2etransfuser","repo_kind":"listed","path":"LetFuser/predict_expert.py","file_url":"https://github.com/pagand/e2etransfuser/blob/HEAD/LetFuser/predict_expert.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6f4eddf7f8def829"}},{"code_sha256_prefix":"a1e4d64b4544efcd","entry":"scale_and_crop_image","repo":"pagand/e2etransfuser","repo_kind":"listed","path":"LetFuser/data.py","file_url":"https://github.com/pagand/e2etransfuser/blob/HEAD/LetFuser/data.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a1e4d64b4544efcd"}},{"code_sha256_prefix":"afd3cd0b6fbb903f","entry":"scale_and_crop_image_cv","repo":"pagand/e2etransfuser","repo_kind":"listed","path":"LetFuser/data.py","file_url":"https://github.com/pagand/e2etransfuser/blob/HEAD/LetFuser/data.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"afd3cd0b6fbb903f"}},{"code_sha256_prefix":"1648fc9d8f6ed785","entry":"swap_RGB2BGR","repo":"pagand/e2etransfuser","repo_kind":"listed","path":"LetFuser/data.py","file_url":"https://github.com/pagand/e2etransfuser/blob/HEAD/LetFuser/data.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1648fc9d8f6ed785"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}