{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/dolphins-multimodal-language-model-for","title":"Dolphins: Multimodal Language Model for Driving","arxiv_id":"2312.00438","date":"2023-12-01","proceeding":null,"authors":["Yingzi Ma","Yulong Cao","Jiachen Sun","Marco Pavone","Chaowei Xiao"],"abstract":"The quest for fully autonomous vehicles (AVs) capable of navigating complex real-world scenarios with human-like understanding and responsiveness. In this paper, we introduce Dolphins, a novel vision-language model architected to imbibe human-like abilities as a conversational driving assistant. Dolphins is adept at processing multimodal inputs comprising video (or image) data, text instructions, and historical control signals to generate informed outputs corresponding to the provided instructions. Building upon the open-sourced pretrained Vision-Language Model, OpenFlamingo, we first enhance Dolphins's reasoning capabilities through an innovative Grounded Chain of Thought (GCoT) process. Then we tailored Dolphins to the driving domain by constructing driving-specific instruction data and conducting instruction tuning. Through the utilization of the BDD-X dataset, we designed and consolidated four distinct AV tasks into Dolphins to foster a holistic understanding of intricate driving scenarios. As a result, the distinctive features of Dolphins are characterized into two dimensions: (1) the ability to provide a comprehensive understanding of complex and long-tailed open-world driving scenarios and solve a spectrum of AV tasks, and (2) the emergence of human-like capabilities including gradient-free instant adaptation via in-context learning and error recovery via reflection.","url_abs":"https://arxiv.org/abs/2312.00438v1","url_pdf":"https://arxiv.org/pdf/2312.00438v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"dolphins-multimodal-language-model-for","repo_url":"https://github.com/safolab-wisc/dolphins","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"dolphins-multimodal-language-model-for","repo_url":"https://github.com/safolab-wisc/fiubench","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"autonomous-vehicles","task_name":"Autonomous Vehicles"},{"task_slug":"in-context-learning","task_name":"In-Context Learning"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"model","task_name":"model"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2312.00438","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.00438"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/safolab-wisc/fiubench","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/safolab-wisc/dolphins","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":1,"unverified":6},"by_repo_kind":{"listed":{"samples":7,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"3614ed4764f65f90","entry":"get_content_type","repo":"safolab-wisc/dolphins","repo_kind":"listed","path":"inference.py","file_url":"https://github.com/safolab-wisc/dolphins/blob/HEAD/inference.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3614ed4764f65f90"}},{"code_sha256_prefix":"103b6da4bd803c70","entry":"add_custom_args","repo":"safolab-wisc/dolphins","repo_kind":"listed","path":"dataset_utils/arguments.py","file_url":"https://github.com/safolab-wisc/dolphins/blob/HEAD/dataset_utils/arguments.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"103b6da4bd803c70"}},{"code_sha256_prefix":"89170c26e2223c98","entry":"add_data_args","repo":"safolab-wisc/dolphins","repo_kind":"listed","path":"dataset_utils/arguments.py","file_url":"https://github.com/safolab-wisc/dolphins/blob/HEAD/dataset_utils/arguments.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"89170c26e2223c98"}},{"code_sha256_prefix":"a0d8d24c812b0c87","entry":"add_training_args","repo":"safolab-wisc/dolphins","repo_kind":"listed","path":"dataset_utils/arguments.py","file_url":"https://github.com/safolab-wisc/dolphins/blob/HEAD/dataset_utils/arguments.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a0d8d24c812b0c87"}},{"code_sha256_prefix":"e67e2f749d91037b","entry":"extract_frames","repo":"safolab-wisc/dolphins","repo_kind":"listed","path":"inference.py","file_url":"https://github.com/safolab-wisc/dolphins/blob/HEAD/inference.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e67e2f749d91037b"}},{"code_sha256_prefix":"dbbd17a289ba66a6","entry":"get_conv_template","repo":"safolab-wisc/dolphins","repo_kind":"listed","path":"conversation.py","file_url":"https://github.com/safolab-wisc/dolphins/blob/HEAD/conversation.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dbbd17a289ba66a6"}},{"code_sha256_prefix":"4aa119881c0ba0a5","entry":"get_image","repo":"safolab-wisc/dolphins","repo_kind":"listed","path":"inference.py","file_url":"https://github.com/safolab-wisc/dolphins/blob/HEAD/inference.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4aa119881c0ba0a5"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}