{"url":"/sota/vision-and-language-navigation-on-vln","task":{"name":"Vision and Language Navigation","url":"/task/vision-and-language-navigation","note":null},"dataset":{"name":"VLN Challenge","url":null},"category":"Robots","categories":["Robots"],"category_note":null,"description":null,"description_from":null,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","rank":"the archive's row order at snapshot; not re-ranked","rows_end_at":"2025-07-28","rows_withheld_as_spam":0,"metric_values":"the archive's strings, untouched"},"metrics":["success","length","error","oracle success","spl"],"metric_direction":{"note":"inferred from the metric name only (the archive records no direction); null = not inferred, chart draws points only","by_metric":{"success":"higher","length":null,"error":"lower","oracle success":"higher","spl":null}},"counts":{"rows":145,"rows_with_code":11,"rows_with_paper_page":14,"rows_dated":14,"rows_using_additional_data":0},"rows":[{"rank_in_archive_order":1,"model":"human","metrics":{"error":"1.61","length":"11.85","oracle success":"0.9","spl":"0.76","success":"0.86"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":2,"model":"Lily","metrics":{"error":"2.5","length":"686.45","oracle success":"0.99","spl":"0.01","success":"0.79"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":3,"model":"Airbert","metrics":{"error":"2.58","length":"686.54","oracle success":"0.99","spl":"0.01","success":"0.78"},"uses_additional_data":false,"paper_date":"2021-08-20","paper":"/paper/airbert-in-domain-pretraining-for-vision-and","paper_url":"https://arxiv.org/abs/2108.09105v1","paper_title":"Airbert: In-domain Pretraining for Vision-and-Language Navigation","code":"https://github.com/jeremylinky/youtube-vln","n_code_links":2,"syntology":{"n_ran":7,"n_unverified":7,"n_samples":14,"n_pointer_only_licence":0}},{"rank_in_archive_order":4,"model":"Global Normalization","metrics":{"error":"2.99","length":"686.86","oracle success":"0.99","spl":"0.01","success":"0.74"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":5,"model":"explore@40  beam-search","metrics":{"error":"3.55","length":"625.27","oracle success":"0.99","spl":"0.01","success":"0.74"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":6,"model":"VLN-Bert","metrics":{"error":"3.09","length":"686.62","oracle success":"0.99","spl":"0.01","success":"0.73"},"uses_additional_data":false,"paper_date":"2020-04-30","paper":"/paper/improving-vision-and-language-navigation-with","paper_url":"https://arxiv.org/abs/2004.14973v2","paper_title":"Improving Vision-and-Language Navigation with Image-Text Pairs from the Web","code":"https://github.com/arjunmajum/vln-bert","n_code_links":1,"syntology":null},{"rank_in_archive_order":7,"model":"BEVBert","metrics":{"error":"3.13","length":"15.87","oracle success":"0.81","spl":"0.62","success":"0.73"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":8,"model":"GMap","metrics":{"error":"3.35","length":"14.43","oracle success":"0.8","spl":"0.62","success":"0.73"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":9,"model":"Gloabl Normalization pre-explore","metrics":{"error":"3.0","length":"10.2","oracle success":"0.8","spl":"0.69","success":"0.73"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":10,"model":"FOAM-Beam Search","metrics":{"error":"3.05","length":"1250.89","oracle success":"1.0","spl":"0.01","success":"0.72"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":11,"model":"Lily","metrics":{"error":"3.44","length":"16.14","oracle success":"0.79","spl":"0.6","success":"0.72"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":12,"model":"ReadNet","metrics":{"error":"3.33","length":"15.24","oracle success":"0.78","spl":"0.61","success":"0.72"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":13,"model":"Active Exploration (Beam Search)","metrics":{"error":"3.07","length":"176.22","oracle success":"0.94","spl":"0.05","success":"0.71"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":14,"model":"Self-Supervised Auxiliary Reasoning Tasks (Beam Search)","metrics":{"error":"3.24","length":"40.85","oracle success":"0.81","spl":"0.21","success":"0.71"},"uses_additional_data":false,"paper_date":"2019-11-18","paper":"/paper/vision-language-navigation-with-self","paper_url":"https://arxiv.org/abs/1911.07883v4","paper_title":"Vision-Language Navigation with Self-Supervised Auxiliary Reasoning Tasks","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":15,"model":"HOC","metrics":{"error":"3.38","length":"15.47","oracle success":"0.79","spl":"0.59","success":"0.71"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":16,"model":"metaexplore","metrics":{"error":"3.57","length":"14.25","oracle success":"0.77","spl":"0.61","success":"0.71"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":17,"model":"sponge","metrics":{"error":"3.26","length":"10.21","oracle success":"0.77","spl":"0.67","success":"0.71"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":18,"model":"SERL (Beam_Search)","metrics":{"error":"3.21","length":"690.61","oracle success":"0.99","spl":"0.01","success":"0.7"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":19,"model":"lxyict","metrics":{"error":"3.61","length":"14.6","oracle success":"0.77","spl":"0.59","success":"0.7"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":20,"model":"DUET+PASTS","metrics":{"error":"3.55","length":"14.39","oracle success":"0.76","spl":"0.6","success":"0.7"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":21,"model":"Single-run","metrics":{"error":"3.52","length":"11.79","oracle success":"0.75","spl":"0.65","success":"0.7"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":22,"model":"Active Exploration (Pre-explore)","metrics":{"error":"3.3","length":"9.85","oracle success":"0.77","spl":"0.68","success":"0.7"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":23,"model":"ADad","metrics":{"error":"3.31","length":"786.35","oracle success":"0.99","spl":"0.01","success":"0.69"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":24,"model":"null","metrics":{"error":"3.26","length":"686.82","oracle success":"0.99","spl":"0.01","success":"0.69"},"uses_additional_data":false,"paper_date":"2019-04-08","paper":"/paper/learning-to-navigate-unseen-environments-back","paper_url":"http://arxiv.org/abs/1904.04195v1","paper_title":"Learning to Navigate Unseen Environments: Back Translation with Environmental Dropout","code":"https://github.com/airsplay/R2R-EnvDrop","n_code_links":1,"syntology":{"n_ran":0,"n_unverified":4,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":25,"model":"CVPR22","metrics":{"error":"3.65","length":"14.73","oracle success":"0.76","spl":"0.59","success":"0.69"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":26,"model":"CMC-AAL2","metrics":{"error":"3.24","length":"11.86","oracle success":"0.76","spl":"0.62","success":"0.69"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":27,"model":"EnvEdit+PT","metrics":{"error":"3.59","length":"11.9","oracle success":"0.73","spl":"0.64","success":"0.68"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":28,"model":"Self-Supervised Auxiliary Reasoning Tasks (Pre-explore)","metrics":{"error":"3.69","length":"10.43","oracle success":"0.75","spl":"0.65","success":"0.68"},"uses_additional_data":false,"paper_date":"2019-11-18","paper":"/paper/vision-language-navigation-with-self","paper_url":"https://arxiv.org/abs/1911.07883v4","paper_title":"Vision-Language Navigation with Self-Supervised Auxiliary Reasoning Tasks","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":29,"model":"DDL","metrics":{"error":"3.74","length":"13.55","oracle success":"0.73","spl":"0.61","success":"0.67"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":30,"model":"CMG-AAL","metrics":{"error":"3.41","length":"12.07","oracle success":"0.76","spl":"0.6","success":"0.67"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":31,"model":"VLN-TreeTrans","metrics":{"error":"3.85","length":"19.62","oracle success":"0.74","spl":"0.55","success":"0.66"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":32,"model":"sliu_team","metrics":{"error":"3.77","length":"16.41","oracle success":"0.71","spl":"0.59","success":"0.66"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":33,"model":"Single-Run, No Pre-Explore","metrics":{"error":"3.73","length":"15.89","oracle success":"0.73","spl":"0.6","success":"0.66"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":34,"model":"TD-STP","metrics":{"error":"3.68","length":"14.78","oracle success":"0.72","spl":"0.6","success":"0.66"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":35,"model":"VLN-BERT-Aug","metrics":{"error":"3.67","length":"13.07","oracle success":"0.73","spl":"0.6","success":"0.66"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":36,"model":"Fortest","metrics":{"error":"3.86","length":"12.75","oracle success":"0.72","spl":"0.6","success":"0.66"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":37,"model":"ESceme Single-run","metrics":{"error":"3.77","length":"11.89","oracle success":"0.72","spl":"0.63","success":"0.66"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":38,"model":"WIN","metrics":{"error":"3.78","length":"15.9","oracle success":"0.71","spl":"0.59","success":"0.65"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":39,"model":"WIN + RecVLN BERT","metrics":{"error":"3.78","length":"15.9","oracle success":"0.71","spl":"0.59","success":"0.65"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":40,"model":"single-run","metrics":{"error":"3.87","length":"13.11","oracle success":"0.72","spl":"0.59","success":"0.65"},"uses_additional_data":false,"paper_date":"2021-06-15","paper":"/paper/vision-language-navigation-with-random","paper_url":"https://arxiv.org/abs/2106.07876v3","paper_title":"Vision-Language Navigation with Random Environmental Mixup","code":"https://github.com/lcfractal/vlnrem","n_code_links":1,"syntology":{"n_ran":1,"n_unverified":3,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":41,"model":"Single-Run","metrics":{"error":"3.82","length":"12.71","oracle success":"0.72","spl":"0.6","success":"0.65"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":42,"model":"HAMT","metrics":{"error":"3.93","length":"12.27","oracle success":"0.72","spl":"0.6","success":"0.65"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":43,"model":"coefficient","metrics":{"error":"3.86","length":"12.22","oracle success":"0.71","spl":"0.61","success":"0.65"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":44,"model":"bin","metrics":{"error":"4.0","length":"11.91","oracle success":"0.7","spl":"0.6","success":"0.65"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":45,"model":"Greedy, No Pre-explore","metrics":{"error":"3.76","length":"10.24","oracle success":"0.71","spl":"0.62","success":"0.65"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":46,"model":"DDL","metrics":{"error":"3.97","length":"13.75","oracle success":"0.71","spl":"0.58","success":"0.64"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":47,"model":"clin","metrics":{"error":"3.9","length":"12.84","oracle success":"0.7","spl":"0.58","success":"0.64"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":48,"model":"single-run","metrics":{"error":"3.88","length":"12.6","oracle success":"0.71","spl":"0.59","success":"0.64"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":49,"model":"PANDA-TingLiu","metrics":{"error":"3.86","length":"12.31","oracle success":"0.71","spl":"0.59","success":"0.64"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":50,"model":"single-run","metrics":{"error":"3.94","length":"12.3","oracle success":"0.71","spl":"0.59","success":"0.64"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":51,"model":"Back Translation with Environmental Dropout (exploring unseen environments before testing)","metrics":{"error":"3.97","length":"9.79","oracle success":"0.7","spl":"0.61","success":"0.64"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":52,"model":"Reinforced Cross-Modal Matching (optimized for SR; with beam search)","metrics":{"error":"4.03","length":"357.62","oracle success":"0.96","spl":"0.02","success":"0.63"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":53,"model":"Hikari","metrics":{"error":"4.0","length":"16.44","oracle success":"0.69","spl":"0.58","success":"0.63"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":54,"model":"SEvol_lzy","metrics":{"error":"4.02","length":"13.57","oracle success":"0.71","spl":"0.57","success":"0.63"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":55,"model":"Colab_buaa","metrics":{"error":"3.98","length":"13.54","oracle success":"0.7","spl":"0.58","success":"0.63"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":56,"model":"Geo","metrics":{"error":"4.04","length":"13.02","oracle success":"0.7","spl":"0.58","success":"0.63"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":57,"model":"YBYB","metrics":{"error":"3.96","length":"12.77","oracle success":"0.7","spl":"0.57","success":"0.63"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":58,"model":"hellohellohello","metrics":{"error":"3.99","length":"12.62","oracle success":"0.71","spl":"0.58","success":"0.63"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":59,"model":"ed","metrics":{"error":"4.16","length":"12.51","oracle success":"0.69","spl":"0.58","success":"0.63"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":60,"model":"Single-Run, No Pre-Explore","metrics":{"error":"4.09","length":"12.35","oracle success":"0.7","spl":"0.57","success":"0.63"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":61,"model":"reg","metrics":{"error":"4.09","length":"12.35","oracle success":"0.7","spl":"0.57","success":"0.63"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":62,"model":"ART","metrics":{"error":"4.09","length":"12.35","oracle success":"0.7","spl":"0.57","success":"0.63"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":63,"model":"binbin","metrics":{"error":"4.07","length":"12.3","oracle success":"0.7","spl":"0.58","success":"0.63"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":64,"model":"CCC(ssm)","metrics":{"error":"4.27","length":"16.94","oracle success":"0.72","spl":"0.49","success":"0.62"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":65,"model":"MARVAL","metrics":{"error":"4.18","length":"10.22","oracle success":"0.67","spl":"0.58","success":"0.62"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":66,"model":"Self-Aware Co-Grounded Model","metrics":{"error":"4.48","length":"373.09","oracle success":"0.97","spl":"0.02","success":"0.61"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":67,"model":"Tactical Rewind - long","metrics":{"error":"4.29","length":"196.53","oracle success":"0.9","spl":"0.03","success":"0.61"},"uses_additional_data":false,"paper_date":"2019-03-06","paper":"/paper/tactical-rewind-self-correction-via","paper_url":"http://arxiv.org/abs/1903.02547v2","paper_title":"Tactical Rewind: Self-Correction via Backtracking in Vision-and-Language Navigation","code":"https://github.com/Kelym/FAST","n_code_links":1,"syntology":{"n_ran":4,"n_unverified":0,"n_samples":4,"n_pointer_only_licence":4}},{"rank_in_archive_order":68,"model":"SSM","metrics":{"error":"4.57","length":"20.39","oracle success":"0.7","spl":"0.46","success":"0.61"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":69,"model":"single-run","metrics":{"error":"4.3","length":"15.83","oracle success":"0.68","spl":"0.55","success":"0.61"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":70,"model":"zq","metrics":{"error":"4.32","length":"13.43","oracle success":"0.69","spl":"0.55","success":"0.61"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":71,"model":"GBSE","metrics":{"error":"4.22","length":"13.2","oracle success":"0.69","spl":"0.56","success":"0.61"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":72,"model":"hellohello","metrics":{"error":"4.26","length":"12.73","oracle success":"0.67","spl":"0.55","success":"0.61"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":73,"model":"homebody","metrics":{"error":"4.27","length":"12.13","oracle success":"0.67","spl":"0.56","success":"0.61"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":74,"model":"GraphBert","metrics":{"error":"4.02","length":"10.66","oracle success":"0.72","spl":"0.55","success":"0.61"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":75,"model":"Single Run","metrics":{"error":"4.34","length":"21.03","oracle success":"0.71","spl":"0.43","success":"0.6"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":76,"model":"SIL-R2","metrics":{"error":"4.21","length":"9.48","oracle success":"0.67","spl":"0.59","success":"0.6"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":77,"model":"Envdrop+SEVol+BT","metrics":{"error":"4.26","length":"14.29","oracle success":"0.66","spl":"0.55","success":"0.59"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":78,"model":"a new baseline","metrics":{"error":"4.53","length":"13.07","oracle success":"0.66","spl":"0.53","success":"0.59"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":79,"model":"trysth","metrics":{"error":"4.49","length":"12.22","oracle success":"0.67","spl":"0.54","success":"0.59"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":80,"model":"SEA features + AuxRN (single-run)","metrics":{"error":"4.71","length":"10.31","oracle success":"0.64","spl":"0.55","success":"0.59"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":81,"model":"PREVALENT","metrics":{"error":"4.52","length":"10.21","oracle success":"0.64","spl":"0.56","success":"0.59"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":82,"model":"SQANv1, No Pre-explore","metrics":{"error":"4.45","length":"13.0","oracle success":"0.67","spl":"0.53","success":"0.58"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":83,"model":"MM2021","metrics":{"error":"4.37","length":"12.98","oracle success":"0.66","spl":"0.54","success":"0.58"},"uses_additional_data":false,"paper_date":"2021-07-15","paper":"/paper/neighbor-view-enhanced-model-for-vision-and","paper_url":"https://arxiv.org/abs/2107.07201v3","paper_title":"Neighbor-view Enhanced Model for Vision and Language Navigation","code":"https://github.com/MarSaKi/NvEM","n_code_links":1,"syntology":{"n_ran":0,"n_unverified":4,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":84,"model":"without pre-explore, beam-search","metrics":{"error":"4.95","length":"10.71","oracle success":"0.65","spl":"0.55","success":"0.58"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":85,"model":"CMG-AAL-TCSVT","metrics":{"error":"4.61","length":"13.16","oracle success":"0.65","spl":"0.5","success":"0.57"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":86,"model":"Single-Run, No Pre-Explore","metrics":{"error":"4.59","length":"12.34","oracle success":"0.65","spl":"0.53","success":"0.57"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":87,"model":"test-sf","metrics":{"error":"4.57","length":"10.99","oracle success":"0.65","spl":"0.5","success":"0.57"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":88,"model":"Greedy","metrics":{"error":"4.53","length":"10.52","oracle success":"0.63","spl":"0.53","success":"0.57"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":89,"model":"null","metrics":{"error":"4.57","length":"1214.94","oracle success":"0.96","spl":"0.01","success":"0.56"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":90,"model":"liuer","metrics":{"error":"4.84","length":"15.74","oracle success":"0.69","spl":"0.48","success":"0.56"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":91,"model":"reward-vln","metrics":{"error":"4.65","length":"12.19","oracle success":"0.62","spl":"0.52","success":"0.56"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":92,"model":"OAG(without pre-explore, beam-search)","metrics":{"error":"5.29","length":"11.39","oracle success":"0.65","spl":"0.53","success":"0.56"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":93,"model":"jmebs","metrics":{"error":"5.17","length":"10.58","oracle success":"0.63","spl":"0.52","success":"0.56"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":94,"model":"SEA features + Env-Dropout (single-run)","metrics":{"error":"4.89","length":"10.18","oracle success":"0.63","spl":"0.53","success":"0.56"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":95,"model":"Single-Run","metrics":{"error":"4.9","length":"12.96","oracle success":"0.62","spl":"0.5","success":"0.55"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":96,"model":"WQ_Pretrain","metrics":{"error":"5.32","length":"10.9","oracle success":"0.63","spl":"0.51","success":"0.55"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":97,"model":"Lang-Vis-Entity VLN (Single-Run)","metrics":{"error":"4.75","length":"10.29","oracle success":"0.61","spl":"0.52","success":"0.55"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":98,"model":"Tactical Rewind - short","metrics":{"error":"5.14","length":"22.08","oracle success":"0.64","spl":"0.41","success":"0.54"},"uses_additional_data":false,"paper_date":"2019-03-06","paper":"/paper/tactical-rewind-self-correction-via","paper_url":"http://arxiv.org/abs/1903.02547v2","paper_title":"Tactical Rewind: Self-Correction via Backtracking in Vision-and-Language Navigation","code":"https://github.com/Kelym/FAST","n_code_links":1,"syntology":{"n_ran":4,"n_unverified":0,"n_samples":4,"n_pointer_only_licence":4}},{"rank_in_archive_order":99,"model":"single run","metrics":{"error":"5.24","length":"14.31","oracle success":"0.64","spl":"0.46","success":"0.54"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":100,"model":"DCMT","metrics":{"error":"4.96","length":"12.51","oracle success":"0.61","spl":"0.48","success":"0.54"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":101,"model":"map1","metrics":{"error":"5.35","length":"11.74","oracle success":"0.64","spl":"0.5","success":"0.54"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":102,"model":"PREVALENT","metrics":{"error":"5.3","length":"10.51","oracle success":"0.61","spl":"0.51","success":"0.54"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":103,"model":"27k","metrics":{"error":"5.11","length":"10.06","oracle success":"0.62","spl":"0.52","success":"0.54"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":104,"model":"Speaker-Follower","metrics":{"error":"4.87","length":"1257.38","oracle success":"0.96","spl":"0.01","success":"0.53"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":105,"model":"GVLN","metrics":{"error":"5.34","length":"15.02","oracle success":"0.61","spl":"0.42","success":"0.53"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":106,"model":"SERL","metrics":{"error":"5.63","length":"12.13","oracle success":"0.61","spl":"0.49","success":"0.53"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":107,"model":"2m-path","metrics":{"error":"5.3","length":"10.4","oracle success":"0.61","spl":"0.5","success":"0.53"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":108,"model":"single-run","metrics":{"error":"5.37","length":"10.0","oracle success":"0.59","spl":"0.5","success":"0.53"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":109,"model":"SYSU-ISE","metrics":{"error":"5.45","length":"10.65","oracle success":"0.6","spl":"0.48","success":"0.52"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":110,"model":"licr19","metrics":{"error":"5.14","length":"13.05","oracle success":"0.6","spl":"0.45","success":"0.51"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":111,"model":"Back Translation with Environmental Dropout (no beam search)","metrics":{"error":"5.23","length":"11.66","oracle success":"0.59","spl":"0.47","success":"0.51"},"uses_additional_data":false,"paper_date":"2019-04-08","paper":"/paper/learning-to-navigate-unseen-environments-back","paper_url":"http://arxiv.org/abs/1904.04195v1","paper_title":"Learning to Navigate Unseen Environments: Back Translation with Environmental Dropout","code":"https://github.com/airsplay/R2R-EnvDrop","n_code_links":1,"syntology":{"n_ran":0,"n_unverified":4,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":112,"model":"SERL (no_augmented)","metrics":{"error":"5.7","length":"11.47","oracle success":"0.57","spl":"0.47","success":"0.51"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":113,"model":"Single-Run, No Pre-Explore","metrics":{"error":"5.45","length":"11.15","oracle success":"0.57","spl":"0.47","success":"0.51"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":114,"model":"testliu","metrics":{"error":"5.5","length":"14.42","oracle success":"0.57","spl":"0.41","success":"0.49"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":115,"model":"Self-Monitoring Navigation Agent (no beam search; Progress Inference)","metrics":{"error":"5.67","length":"18.04","oracle success":"0.59","spl":"0.35","success":"0.48"},"uses_additional_data":false,"paper_date":"2019-01-10","paper":"/paper/self-monitoring-navigation-agent-via","paper_url":"http://arxiv.org/abs/1901.03035v1","paper_title":"Self-Monitoring Navigation Agent via Auxiliary Progress Estimation","code":"https://github.com/chihyaoma/selfmonitoring-agent","n_code_links":2,"syntology":{"n_ran":0,"n_unverified":6,"n_samples":6,"n_pointer_only_licence":0}},{"rank_in_archive_order":116,"model":"The Regretful Agent (no beam search; greedy action selection)","metrics":{"error":"5.69","length":"13.69","oracle success":"0.56","spl":"0.4","success":"0.48"},"uses_additional_data":false,"paper_date":"2019-03-05","paper":"/paper/the-regretful-agent-heuristic-aided","paper_url":"http://arxiv.org/abs/1903.01602v1","paper_title":"The Regretful Agent: Heuristic-Aided Navigation through Progress Estimation","code":"https://github.com/chihyaoma/regretful-agent","n_code_links":3,"syntology":{"n_ran":5,"n_unverified":6,"n_samples":11,"n_pointer_only_licence":0}},{"rank_in_archive_order":117,"model":"ALTR","metrics":{"error":"5.49","length":"10.27","oracle success":"0.56","spl":"0.45","success":"0.48"},"uses_additional_data":false,"paper_date":"2019-08-09","paper":"/paper/transferable-representation-learning-in","paper_url":"https://arxiv.org/abs/1908.03409v2","paper_title":"Transferable Representation Learning in Vision-and-Language Navigation","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":118,"model":"selfmoni0","metrics":{"error":"5.8","length":"16.73","oracle success":"0.57","spl":"0.36","success":"0.47"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":119,"model":"DoubleAttn","metrics":{"error":"5.56","length":"16.03","oracle success":"0.57","spl":"0.35","success":"0.47"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":120,"model":"SEA features + Speaker-Follower (single-run)","metrics":{"error":"5.42","length":"14.07","oracle success":"0.55","spl":"0.4","success":"0.47"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":121,"model":"naive","metrics":{"error":"5.64","length":"10.42","oracle success":"0.53","spl":"0.43","success":"0.47"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":122,"model":"Environment-Agnostic Multitask Learning","metrics":{"error":"6.03","length":"13.35","oracle success":"0.56","spl":"0.4","success":"0.45"},"uses_additional_data":false,"paper_date":"2020-03-01","paper":"/paper/environment-agnostic-multitask-learning-for","paper_url":"https://arxiv.org/abs/2003.00443v5","paper_title":"Environment-agnostic Multitask Learning for Natural Language Grounded Navigation","code":"https://github.com/google-research/valan","n_code_links":1,"syntology":null},{"rank_in_archive_order":123,"model":"speaker_follower_tesk2","metrics":{"error":"6.01","length":"10.47","oracle success":"0.53","spl":"0.4","success":"0.45"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":124,"model":"Reinforced Cross-Modal Matching (single trajectory; NO beam search)","metrics":{"error":"6.12","length":"11.97","oracle success":"0.5","spl":"0.38","success":"0.43"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":125,"model":"PTA","metrics":{"error":"6.17","length":"10.17","oracle success":"0.47","spl":"0.36","success":"0.4"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":126,"model":"Khanh Nguyen","metrics":{"error":"6.46","length":"13.08","oracle success":"0.45","spl":"0.3","success":"0.37"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":127,"model":"HAIL","metrics":{"error":"6.61","length":"13.85","oracle success":"0.46","spl":"0.3","success":"0.36"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":128,"model":"rcm_test","metrics":{"error":"6.67","length":"10.83","oracle success":"0.44","spl":"0.33","success":"0.36"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":129,"model":"ai like samurai with PNasNet5Large","metrics":{"error":"6.95","length":"10.44","oracle success":"0.43","spl":"0.31","success":"0.36"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":130,"model":"Dynamic Convolutional Filters","metrics":{"error":"6.55","length":"9.81","oracle success":"0.45","spl":"0.31","success":"0.35"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":131,"model":"AnonymousTeam","metrics":{"error":"6.89","length":"8.32","oracle success":"0.41","spl":"0.32","success":"0.34"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":132,"model":"base_1","metrics":{"error":"6.68","length":"15.75","oracle success":"0.43","spl":"0.25","success":"0.33"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":133,"model":"fuse_1","metrics":{"error":"7.03","length":"15.1","oracle success":"0.4","spl":"0.25","success":"0.31"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":134,"model":"no","metrics":{"error":"7.63","length":"22.14","oracle success":"0.61","spl":"0.2","success":"0.3"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":135,"model":"base_0","metrics":{"error":"7.2","length":"15.9","oracle success":"0.4","spl":"0.23","success":"0.29"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":136,"model":"test_an","metrics":{"error":"7.47","length":"10.92","oracle success":"0.34","spl":"0.21","success":"0.26"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":137,"model":"Look Before You Leap","metrics":{"error":"7.53","length":"9.15","oracle success":"0.32","spl":"0.23","success":"0.25"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":138,"model":"X-Modal","metrics":{"error":"8.56","length":"9.48","oracle success":"0.32","spl":"0.22","success":"0.24"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":139,"model":"zhangyong","metrics":{"error":"7.99","length":"8.26","oracle success":"0.26","spl":"0.18","success":"0.2"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":140,"model":"Seq2Seq Baseline","metrics":{"error":"7.85","length":"8.13","oracle success":"0.27","spl":"0.18","success":"0.2"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":141,"model":"zy123","metrics":{"error":"8.82","length":"9.56","oracle success":"0.24","spl":"0.16","success":"0.18"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":142,"model":"zzzzzzzz55768","metrics":{"error":"9.8","length":"9.91","oracle success":"0.19","spl":"0.12","success":"0.14"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":143,"model":"Random Agent","metrics":{"error":"9.79","length":"9.89","oracle success":"0.18","spl":"0.12","success":"0.13"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":144,"model":"1111","metrics":{"error":"12.29","length":"45.13","oracle success":"0.49","spl":"0.02","success":"0.07"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":145,"model":"15458","metrics":{"error":"9.93","length":"0.0","oracle success":"0.0","spl":"0.0","success":"0.0"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null}],"since_archive":{"claim":"Results that newer papers report for their own method, placed here by Syntology. A model pointed at the cell in the paper's own table; the number was read from that cell and checked against this leaderboard's metric, dataset, split and scale; an independent check that saw this leaderboard's other rows and every other leaderboard on the same dataset accepted it. Not reviewed by the paper's authors or by the archive's editors, and not ranked against the archive rows.","extraction_file_present":true,"measurement":{"test_papers":883,"papers_with_output":881,"judged_true":108,"judged":110,"wilson95_lower":0.9361,"measured_on":"2026-09-24","frozen_commit":"0e3de0df94"},"measurement_note":"blind adjudication of accepted entries on a held-out split of archive papers, rules frozen before the test","coverage":{"sentence":"Syntology has checked 6,885 of the 9,623 papers on this site that are newer than the archive; results from the others appear after they are checked.","complete":false,"papers_newer_than_archive":9623,"papers_checked":6885,"papers_extracted_not_yet_verified":0,"boards_without_verdict":2,"papers_not_yet_extracted":2737},"order":"newest first by month (arXiv date, else the arXiv-id month), then arXiv id descending","columns":[],"entries":[]},"syntology":{"read_at":"2026-09-25T09:33:49+00:00","claim":"Per row: N of M harvested code samples from that row's paper executed on a synthesized fixture; the other M-N are unverified. Not a reproduction of the row's number; not a correctness claim. n_pointer_only_licence counts samples the site points at rather than redistributes (a licence axis, independent of ran/unverified).","rows_with_graph_line":9,"rows_with_any_sample_ran":5,"distinct_papers_with_graph_line":7,"distinct_papers_with_any_sample_ran":4,"samples_over_distinct_papers":{"n_ran":17,"n_unverified":30,"n_samples":47,"n_pointer_only_licence":4,"note":"each paper (arXiv id) counted once, however many rows it is behind; this is the page-level figure"},"samples_row_weighted":{"n_ran":21,"n_unverified":34,"n_samples":55,"n_pointer_only_licence":8,"note":"row-weighted: a paper behind several rows is counted once per row; inflated relative to samples_over_distinct_papers by design, kept for readers summing the per-row syntology blocks"}}}