{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/flame-learning-to-navigate-with-multimodal","title":"FLAME: Learning to Navigate with Multimodal LLM in Urban Environments","arxiv_id":"2408.11051","date":"2024-08-20","proceeding":null,"authors":["Yunzhe Xu","Yiyuan Pan","Zhe Liu","Hesheng Wang"],"abstract":"Large Language Models (LLMs) have demonstrated potential in Vision-and-Language Navigation (VLN) tasks, yet current applications face challenges. While LLMs excel in general conversation scenarios, they struggle with specialized navigation tasks, yielding suboptimal performance compared to specialized VLN models. We introduce FLAME (FLAMingo-Architected Embodied Agent), a novel Multimodal LLM-based agent and architecture designed for urban VLN tasks that efficiently handles multiple observations. Our approach implements a three-phase tuning technique for effective adaptation to navigation tasks, including single perception tuning for street view description, multiple perception tuning for route summarization, and end-to-end training on VLN datasets. The augmented datasets are synthesized automatically. Experimental results demonstrate FLAME's superiority over existing methods, surpassing state-of-the-art methods by a 7.3% increase in task completion on Touchdown dataset. This work showcases the potential of Multimodal LLMs (MLLMs) in complex navigation tasks, representing an advancement towards applications of MLLMs in the field of embodied intelligence.","url_abs":"https://arxiv.org/abs/2408.11051v2","url_pdf":"https://arxiv.org/pdf/2408.11051v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"flame-learning-to-navigate-with-multimodal","repo_url":"https://github.com/xyz9911/FLAME","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"navigate","task_name":"Navigate"},{"task_slug":"vision-and-language-navigation","task_name":"Vision and Language Navigation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/vision-and-language-navigation-on-touchdown","task":"Vision and Language Navigation","dataset":"Touchdown Dataset","model":"FLAME","rank_in_archive_order":1,"of":12,"metrics":{"Task Completion (TC)":"40.20"},"uses_additional_data":false},{"leaderboard":"/sota/vision-and-language-navigation-on-map2seq","task":"Vision and Language Navigation","dataset":"map2seq","model":"FLAME","rank_in_archive_order":1,"of":5,"metrics":{"Task Completion (TC)":"52.44"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2408.11051","atlas_url":"https://app.syntology.ai/?focus=2408.11051","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2408.11051"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/xyz9911/FLAME","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_fixture":2,"ran_violates":1,"ran_draft_wrong":2,"ran":1,"unverified":1},"by_repo_kind":{"official":{"samples":7,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"30d7eec482ebf6b1","entry":"repeat_kv","repo":"xyz9911/FLAME","repo_kind":"official","path":"llm_nav/model/llama_xformer.py","file_url":"https://github.com/xyz9911/FLAME/blob/HEAD/llm_nav/model/llama_xformer.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":2,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"30d7eec482ebf6b1"}},{"code_sha256_prefix":"f725bc2d76076485","entry":"apply_rotary_pos_emb","repo":"xyz9911/FLAME","repo_kind":"official","path":"llm_nav/model/llama_xformer.py","file_url":"https://github.com/xyz9911/FLAME/blob/HEAD/llm_nav/model/llama_xformer.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f725bc2d76076485"}},{"code_sha256_prefix":"aa5486a3650902d8","entry":"exists","repo":"xyz9911/FLAME","repo_kind":"official","path":"llm_nav/model/modeling_flamingo.py","file_url":"https://github.com/xyz9911/FLAME/blob/HEAD/llm_nav/model/modeling_flamingo.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"aa5486a3650902d8"}},{"code_sha256_prefix":"fe68fff801021b23","entry":"getattr_recursive","repo":"xyz9911/FLAME","repo_kind":"official","path":"llm_nav/model/modeling_flamingo.py","file_url":"https://github.com/xyz9911/FLAME/blob/HEAD/llm_nav/model/modeling_flamingo.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fe68fff801021b23"}},{"code_sha256_prefix":"37b19b055de49001","entry":"inference","repo":"xyz9911/FLAME","repo_kind":"official","path":"llm_nav/agent.py","file_url":"https://github.com/xyz9911/FLAME/blob/HEAD/llm_nav/agent.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"37b19b055de49001"}},{"code_sha256_prefix":"b99eea6376d1e212","entry":"rotate_half","repo":"xyz9911/FLAME","repo_kind":"official","path":"llm_nav/model/llama_xformer.py","file_url":"https://github.com/xyz9911/FLAME/blob/HEAD/llm_nav/model/llama_xformer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b99eea6376d1e212"}},{"code_sha256_prefix":"54b2d55aa476a3bc","entry":"unwrap_model","repo":"xyz9911/FLAME","repo_kind":"official","path":"llm_nav/trainer.py","file_url":"https://github.com/xyz9911/FLAME/blob/HEAD/llm_nav/trainer.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"54b2d55aa476a3bc"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}