{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/airbert-in-domain-pretraining-for-vision-and","title":"Airbert: In-domain Pretraining for Vision-and-Language Navigation","arxiv_id":"2108.09105","date":"2021-08-20","proceeding":"ICCV 2021 10","authors":["Pierre-Louis Guhur","Makarand Tapaswi","ShiZhe Chen","Ivan Laptev","Cordelia Schmid"],"abstract":"Vision-and-language navigation (VLN) aims to enable embodied agents to navigate in realistic environments using natural language instructions. Given the scarcity of domain-specific training data and the high diversity of image and language inputs, the generalization of VLN agents to unseen environments remains challenging. Recent methods explore pretraining to improve generalization, however, the use of generic image-caption datasets or existing small-scale VLN environments is suboptimal and results in limited improvements. In this work, we introduce BnB, a large-scale and diverse in-domain VLN dataset. We first collect image-caption (IC) pairs from hundreds of thousands of listings from online rental marketplaces. Using IC pairs we next propose automatic strategies to generate millions of VLN path-instruction (PI) pairs. We further propose a shuffling loss that improves the learning of temporal order inside PI pairs. We use BnB pretrain our Airbert model that can be adapted to discriminative and generative settings and show that it outperforms state of the art for Room-to-Room (R2R) navigation and Remote Referring Expression (REVERIE) benchmarks. Moreover, our in-domain pretraining significantly increases performance on a challenging few-shot VLN evaluation, where we train the model only on VLN instructions from a few houses.","url_abs":"https://arxiv.org/abs/2108.09105v1","url_pdf":"https://arxiv.org/pdf/2108.09105v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"airbert-in-domain-pretraining-for-vision-and","repo_url":"https://github.com/airbert-vln/airbert","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"airbert-in-domain-pretraining-for-vision-and","repo_url":"https://github.com/jeremylinky/youtube-vln","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"navigate","task_name":"Navigate"},{"task_slug":"referring-expression","task_name":"Referring Expression"},{"task_slug":"vision-and-language-navigation","task_name":"Vision and Language Navigation"}],"methods":[],"datasets_introduced":[{"slug":"bnb","name":"BnB","full_name":""}],"methods_introduced":[],"results":[{"leaderboard":"/sota/vision-and-language-navigation-on-vln","task":"Vision and Language Navigation","dataset":"VLN Challenge","model":"Airbert","rank_in_archive_order":3,"of":145,"metrics":{"error":"2.58","length":"686.54","oracle success":"0.99","spl":"0.01","success":"0.78"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2108.09105","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2108.09105"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/airbert-vln/airbert","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jeremylinky/youtube-vln","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":2,"ran":4,"ran_honours":1,"unverified":7},"by_repo_kind":{"official":{"samples":14,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"0f786c407fb1ee4c","entry":"swish","repo":"airbert-vln/airbert","repo_kind":"official","path":"vilbert/vilbert.py","file_url":"https://github.com/airbert-vln/airbert/blob/HEAD/vilbert/vilbert.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":2,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0f786c407fb1ee4c"}},{"code_sha256_prefix":"c58840db809196b1","entry":"cached_path","repo":"airbert-vln/airbert","repo_kind":"official","path":"vilbert/file_utils.py","file_url":"https://github.com/airbert-vln/airbert/blob/HEAD/vilbert/file_utils.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c58840db809196b1"}},{"code_sha256_prefix":"e325858dc74dac65","entry":"filename_to_url","repo":"airbert-vln/airbert","repo_kind":"official","path":"vilbert/file_utils.py","file_url":"https://github.com/airbert-vln/airbert/blob/HEAD/vilbert/file_utils.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e325858dc74dac65"}},{"code_sha256_prefix":"fdc64f4c72036ae4","entry":"gelu","repo":"airbert-vln/airbert","repo_kind":"official","path":"vilbert/vilbert.py","file_url":"https://github.com/airbert-vln/airbert/blob/HEAD/vilbert/vilbert.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fdc64f4c72036ae4"}},{"code_sha256_prefix":"67de8cef340cc324","entry":"get_world_size","repo":"airbert-vln/airbert","repo_kind":"official","path":"utils/distributed.py","file_url":"https://github.com/airbert-vln/airbert/blob/HEAD/utils/distributed.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"67de8cef340cc324"}},{"code_sha256_prefix":"45e8b849b6a9b735","entry":"is_default_gpu","repo":"airbert-vln/airbert","repo_kind":"official","path":"utils/misc.py","file_url":"https://github.com/airbert-vln/airbert/blob/HEAD/utils/misc.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"45e8b849b6a9b735"}},{"code_sha256_prefix":"af64ec220e8bcdbc","entry":"url_to_filename","repo":"airbert-vln/airbert","repo_kind":"official","path":"vilbert/file_utils.py","file_url":"https://github.com/airbert-vln/airbert/blob/HEAD/vilbert/file_utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"af64ec220e8bcdbc"}},{"code_sha256_prefix":"4803e9d0f2a1f293","entry":"get_instr_length","repo":"airbert-vln/airbert","repo_kind":"official","path":"train_speaker.py","file_url":"https://github.com/airbert-vln/airbert/blob/HEAD/train_speaker.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4803e9d0f2a1f293"}},{"code_sha256_prefix":"34279e5b4ed2577f","entry":"get_logger","repo":"airbert-vln/airbert","repo_kind":"official","path":"utils/misc.py","file_url":"https://github.com/airbert-vln/airbert/blob/HEAD/utils/misc.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"34279e5b4ed2577f"}},{"code_sha256_prefix":"c899935328a33c24","entry":"get_output_dir","repo":"airbert-vln/airbert","repo_kind":"official","path":"utils/misc.py","file_url":"https://github.com/airbert-vln/airbert/blob/HEAD/utils/misc.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c899935328a33c24"}},{"code_sha256_prefix":"8d57e6390318ac16","entry":"get_rank","repo":"airbert-vln/airbert","repo_kind":"official","path":"utils/distributed.py","file_url":"https://github.com/airbert-vln/airbert/blob/HEAD/utils/distributed.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8d57e6390318ac16"}},{"code_sha256_prefix":"baa5766f4566aafc","entry":"load_tf_weights_in_bert","repo":"airbert-vln/airbert","repo_kind":"official","path":"vilbert/vilbert.py","file_url":"https://github.com/airbert-vln/airbert/blob/HEAD/vilbert/vilbert.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"baa5766f4566aafc"}},{"code_sha256_prefix":"9e8292655b4f13cb","entry":"rollout","repo":"airbert-vln/airbert","repo_kind":"official","path":"train_speaker.py","file_url":"https://github.com/airbert-vln/airbert/blob/HEAD/train_speaker.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9e8292655b4f13cb"}},{"code_sha256_prefix":"9fcc9f24c53b690d","entry":"val_epoch","repo":"airbert-vln/airbert","repo_kind":"official","path":"train_speaker.py","file_url":"https://github.com/airbert-vln/airbert/blob/HEAD/train_speaker.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9fcc9f24c53b690d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}