{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/drivemllm-a-benchmark-for-spatial","title":"DriveMLLM: A Benchmark for Spatial Understanding with Multimodal Large Language Models in Autonomous Driving","arxiv_id":"2411.13112","date":"2024-11-20","proceeding":null,"authors":["Xianda Guo","Ruijun Zhang","Yiqun Duan","Yuhang He","Chenming Zhang","Shuai Liu","Long Chen"],"abstract":"Autonomous driving requires a comprehensive understanding of 3D environments to facilitate high-level tasks such as motion prediction, planning, and mapping. In this paper, we introduce DriveMLLM, a benchmark specifically designed to evaluate the spatial understanding capabilities of multimodal large language models (MLLMs) in autonomous driving. DriveMLLM includes 880 front-facing camera images and introduces both absolute and relative spatial reasoning tasks, accompanied by linguistically diverse natural language questions. To measure MLLMs' performance, we propose novel evaluation metrics focusing on spatial understanding. We evaluate several state-of-the-art MLLMs on DriveMLLM, and our results reveal the limitations of current models in understanding complex spatial relationships in driving contexts. We believe these findings underscore the need for more advanced MLLM-based spatial reasoning methods and highlight the potential for DriveMLLM to drive further research in autonomous driving. Code will be available at \\url{https://github.com/XiandaGuo/Drive-MLLM}.","url_abs":"https://arxiv.org/abs/2411.13112v2","url_pdf":"https://arxiv.org/pdf/2411.13112v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"drivemllm-a-benchmark-for-spatial","repo_url":"https://github.com/xiandaguo/drive-mllm","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"autonomous-driving","task_name":"Autonomous Driving"},{"task_slug":"spatial-reasoning","task_name":"Spatial Reasoning"},{"task_slug":"motion-prediction","task_name":"motion prediction"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2411.13112","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2411.13112"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/xiandaguo/drive-mllm","reach":null}],"summary":{"ran_violates":1,"ran_honours":1,"ran_fixture":1,"ran_draft_wrong":1},"by_repo_kind":{"official":{"samples":4,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":4,"samples":[{"code_sha256_prefix":"23d0e56a5a85e59d","entry":"check_correct","repo":"xiandaguo/drive-mllm","repo_kind":"official","path":"training/grpo/grpo.py","file_url":"https://github.com/xiandaguo/drive-mllm/blob/HEAD/training/grpo/grpo.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"23d0e56a5a85e59d"}},{"code_sha256_prefix":"bbd19315c49635ff","entry":"get_score","repo":"xiandaguo/drive-mllm","repo_kind":"official","path":"evaluation/eval_from_json.py","file_url":"https://github.com/xiandaguo/drive-mllm/blob/HEAD/evaluation/eval_from_json.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bbd19315c49635ff"}},{"code_sha256_prefix":"0660217a2006f431","entry":"iou","repo":"xiandaguo/drive-mllm","repo_kind":"official","path":"evaluation/eval_from_json.py","file_url":"https://github.com/xiandaguo/drive-mllm/blob/HEAD/evaluation/eval_from_json.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0660217a2006f431"}},{"code_sha256_prefix":"4e2023c1a285d572","entry":"normalize","repo":"xiandaguo/drive-mllm","repo_kind":"official","path":"training/grpo/grpo.py","file_url":"https://github.com/xiandaguo/drive-mllm/blob/HEAD/training/grpo/grpo.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4e2023c1a285d572"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}