{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/think-global-act-local-dual-scale-graph","title":"Think Global, Act Local: Dual-scale Graph Transformer for Vision-and-Language Navigation","arxiv_id":"2202.11742","date":"2022-02-23","proceeding":"CVPR 2022 1","authors":["ShiZhe Chen","Pierre-Louis Guhur","Makarand Tapaswi","Cordelia Schmid","Ivan Laptev"],"abstract":"Following language instructions to navigate in unseen environments is a challenging problem for autonomous embodied agents. The agent not only needs to ground languages in visual scenes, but also should explore the environment to reach its target. In this work, we propose a dual-scale graph transformer (DUET) for joint long-term action planning and fine-grained cross-modal understanding. We build a topological map on-the-fly to enable efficient exploration in global action space. To balance the complexity of large action space reasoning and fine-grained language grounding, we dynamically combine a fine-scale encoding over local observations and a coarse-scale encoding on a global map via graph transformers. The proposed approach, DUET, significantly outperforms state-of-the-art methods on goal-oriented vision-and-language navigation (VLN) benchmarks REVERIE and SOON. It also improves the success rate on the fine-grained VLN benchmark R2R.","url_abs":"https://arxiv.org/abs/2202.11742v1","url_pdf":"https://arxiv.org/pdf/2202.11742v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"think-global-act-local-dual-scale-graph","repo_url":"https://github.com/cshizhe/vln-duet","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"efficient-exploration","task_name":"Efficient Exploration"},{"task_slug":"navigate","task_name":"Navigate"},{"task_slug":"vision-and-language-navigation","task_name":"Vision and Language Navigation"},{"task_slug":"visual-navigation","task_name":"Visual Navigation"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"graph-transformer","method_name":"Graph Transformer"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"lapeigen","method_name":"LapEigen"},{"method_slug":"laplacian-pe","method_name":"Laplacian PE"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/visual-navigation-on-room-to-room-1","task":"Visual Navigation","dataset":"R2R","model":"DUET","rank_in_archive_order":7,"of":11,"metrics":{"spl":"0.58"},"uses_additional_data":false},{"leaderboard":"/sota/visual-navigation-on-soon-test","task":"Visual Navigation","dataset":"SOON Test","model":"DUET","rank_in_archive_order":5,"of":6,"metrics":{"Nav-SPL":"21.42","SR":"33.44"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2202.11742","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2202.11742"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/cshizhe/vln-duet","reach":null}],"summary":{"ran":1,"ran_violates":1,"ran_fixture":2,"unverified":1},"by_repo_kind":{"listed":{"samples":5,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":5,"samples":[{"code_sha256_prefix":"5aa223bfc228289c","entry":"FloydGraph","repo":"cshizhe/vln-duet","repo_kind":"listed","path":"map_nav_src/models/graph_utils.py","file_url":"https://github.com/cshizhe/vln-duet/blob/HEAD/map_nav_src/models/graph_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5aa223bfc228289c"}},{"code_sha256_prefix":"c362aaed2935ceff","entry":"calc_position_distance","repo":"cshizhe/vln-duet","repo_kind":"listed","path":"map_nav_src/models/graph_utils.py","file_url":"https://github.com/cshizhe/vln-duet/blob/HEAD/map_nav_src/models/graph_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c362aaed2935ceff"}},{"code_sha256_prefix":"168e35f8ed1ad0ce","entry":"calculate_vp_rel_pos_fts","repo":"cshizhe/vln-duet","repo_kind":"listed","path":"map_nav_src/models/graph_utils.py","file_url":"https://github.com/cshizhe/vln-duet/blob/HEAD/map_nav_src/models/graph_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"168e35f8ed1ad0ce"}},{"code_sha256_prefix":"dac1993f5ce8e12e","entry":"get_angle_fts","repo":"cshizhe/vln-duet","repo_kind":"listed","path":"map_nav_src/models/graph_utils.py","file_url":"https://github.com/cshizhe/vln-duet/blob/HEAD/map_nav_src/models/graph_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"dac1993f5ce8e12e"}},{"code_sha256_prefix":"539291c811bbfd63","entry":"GraphMap","repo":"cshizhe/vln-duet","repo_kind":"listed","path":"map_nav_src/models/graph_utils.py","file_url":"https://github.com/cshizhe/vln-duet/blob/HEAD/map_nav_src/models/graph_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"539291c811bbfd63"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}