{"url":"/sota/conversational-web-navigation-on-weblinx","task":{"name":"Conversational Web Navigation","url":"/task/conversational-web-navigation","note":null},"dataset":{"name":"WebLINX","url":"/dataset/weblinx"},"category":"Natural Language Processing","categories":["Natural Language Processing"],"category_note":null,"description":"The problem of conversational web navigation is described as follow: a digital agent controls a web browser and follows user instructions to solve real-world tasks in a multi-turn dialogue fashion. It was introduced alongside the WebLINX benchmark (Lù, Kasner, Reddy, 2024), and complements tasks such as Autonomous Web Navigation. It is one of many problems tackled by generalist (web) agents.","description_from":"task","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","rank":"the archive's row order at snapshot; not re-ranked","rows_end_at":"2025-07-28","rows_withheld_as_spam":0,"metric_values":"the archive's strings, untouched"},"metrics":["Overall score","Intent Match","Element (IoU)","Text (F1)"],"metric_direction":{"note":"inferred from the metric name only (the archive records no direction); null = not inferred, chart draws points only","by_metric":{"Overall score":"higher","Intent Match":null,"Element (IoU)":"higher","Text (F1)":"higher"}},"counts":{"rows":17,"rows_with_code":17,"rows_with_paper_page":17,"rows_dated":17,"rows_using_additional_data":0},"rows":[{"rank_in_archive_order":1,"model":"Llama-2-13B","metrics":{"Element (IoU)":"22.82","Intent Match":"81.91","Overall score":"25.21","Text (F1)":"26.60"},"uses_additional_data":false,"paper_date":"2024-02-08","paper":"/paper/weblinx-real-world-website-navigation-with","paper_url":"https://arxiv.org/abs/2402.05930v2","paper_title":"WebLINX: Real-World Website Navigation with Multi-Turn Dialogue","code":"https://github.com/McGill-NLP/webllama","n_code_links":2,"syntology":{"n_ran":21,"n_unverified":6,"n_samples":27,"n_pointer_only_licence":0}},{"rank_in_archive_order":2,"model":"S-LLaMA-2.7B","metrics":{"Element (IoU)":"22.60","Intent Match":"84.00","Overall score":"25.02","Text (F1)":"27.17"},"uses_additional_data":false,"paper_date":"2024-02-08","paper":"/paper/weblinx-real-world-website-navigation-with","paper_url":"https://arxiv.org/abs/2402.05930v2","paper_title":"WebLINX: Real-World Website Navigation with Multi-Turn Dialogue","code":"https://github.com/McGill-NLP/webllama","n_code_links":2,"syntology":{"n_ran":21,"n_unverified":6,"n_samples":27,"n_pointer_only_licence":0}},{"rank_in_archive_order":3,"model":"Llama-2-7B","metrics":{"Element (IoU)":"22.26","Intent Match":"82.64","Overall score":"24.57","Text (F1)":"26.50"},"uses_additional_data":false,"paper_date":"2024-02-08","paper":"/paper/weblinx-real-world-website-navigation-with","paper_url":"https://arxiv.org/abs/2402.05930v2","paper_title":"WebLINX: Real-World Website Navigation with Multi-Turn Dialogue","code":"https://github.com/McGill-NLP/webllama","n_code_links":2,"syntology":{"n_ran":21,"n_unverified":6,"n_samples":27,"n_pointer_only_licence":0}},{"rank_in_archive_order":4,"model":"Flan-T5-3B","metrics":{"Element (IoU)":"20.31","Intent Match":"81.14","Overall score":"23.77","Text (F1)":"25.75"},"uses_additional_data":false,"paper_date":"2024-02-08","paper":"/paper/weblinx-real-world-website-navigation-with","paper_url":"https://arxiv.org/abs/2402.05930v2","paper_title":"WebLINX: Real-World Website Navigation with Multi-Turn Dialogue","code":"https://github.com/McGill-NLP/webllama","n_code_links":2,"syntology":{"n_ran":21,"n_unverified":6,"n_samples":27,"n_pointer_only_licence":0}},{"rank_in_archive_order":5,"model":"S-LLaMA-1.3B","metrics":{"Element (IoU)":"20.54","Intent Match":"83.32","Overall score":"23.73","Text (F1)":"25.85"},"uses_additional_data":false,"paper_date":"2024-02-08","paper":"/paper/weblinx-real-world-website-navigation-with","paper_url":"https://arxiv.org/abs/2402.05930v2","paper_title":"WebLINX: Real-World Website Navigation with Multi-Turn Dialogue","code":"https://github.com/McGill-NLP/webllama","n_code_links":2,"syntology":{"n_ran":21,"n_unverified":6,"n_samples":27,"n_pointer_only_licence":0}},{"rank_in_archive_order":6,"model":"GPT-3.5F","metrics":{"Element (IoU)":"18.64","Intent Match":"77.56","Overall score":"21.22","Text (F1)":"22.39"},"uses_additional_data":false,"paper_date":"2024-02-08","paper":"/paper/weblinx-real-world-website-navigation-with","paper_url":"https://arxiv.org/abs/2402.05930v2","paper_title":"WebLINX: Real-World Website Navigation with Multi-Turn Dialogue","code":"https://github.com/McGill-NLP/webllama","n_code_links":2,"syntology":{"n_ran":21,"n_unverified":6,"n_samples":27,"n_pointer_only_licence":0}},{"rank_in_archive_order":7,"model":"MindAct-3B","metrics":{"Element (IoU)":"16.50","Intent Match":"79.89","Overall score":"20.94","Text (F1)":"23.16"},"uses_additional_data":false,"paper_date":"2024-02-08","paper":"/paper/weblinx-real-world-website-navigation-with","paper_url":"https://arxiv.org/abs/2402.05930v2","paper_title":"WebLINX: Real-World Website Navigation with Multi-Turn Dialogue","code":"https://github.com/McGill-NLP/webllama","n_code_links":2,"syntology":{"n_ran":21,"n_unverified":6,"n_samples":27,"n_pointer_only_licence":0}},{"rank_in_archive_order":8,"model":"Fuyu-8B","metrics":{"Element (IoU)":"15.70","Intent Match":"80.07","Overall score":"19.97","Text (F1)":"22.30"},"uses_additional_data":false,"paper_date":"2024-02-08","paper":"/paper/weblinx-real-world-website-navigation-with","paper_url":"https://arxiv.org/abs/2402.05930v2","paper_title":"WebLINX: Real-World Website Navigation with Multi-Turn Dialogue","code":"https://github.com/McGill-NLP/webllama","n_code_links":2,"syntology":{"n_ran":21,"n_unverified":6,"n_samples":27,"n_pointer_only_licence":0}},{"rank_in_archive_order":9,"model":"Flan-T5-780M","metrics":{"Element (IoU)":"15.36","Intent Match":"80.02","Overall score":"17.27","Text (F1)":"14.05"},"uses_additional_data":false,"paper_date":"2024-02-08","paper":"/paper/weblinx-real-world-website-navigation-with","paper_url":"https://arxiv.org/abs/2402.05930v2","paper_title":"WebLINX: Real-World Website Navigation with Multi-Turn Dialogue","code":"https://github.com/McGill-NLP/webllama","n_code_links":2,"syntology":{"n_ran":21,"n_unverified":6,"n_samples":27,"n_pointer_only_licence":0}},{"rank_in_archive_order":10,"model":"Pix2Act-1.3B","metrics":{"Element (IoU)":"8.28","Intent Match":"81.80","Overall score":"16.88","Text (F1)":"25.21"},"uses_additional_data":false,"paper_date":"2024-02-08","paper":"/paper/weblinx-real-world-website-navigation-with","paper_url":"https://arxiv.org/abs/2402.05930v2","paper_title":"WebLINX: Real-World Website Navigation with Multi-Turn Dialogue","code":"https://github.com/McGill-NLP/webllama","n_code_links":2,"syntology":{"n_ran":21,"n_unverified":6,"n_samples":27,"n_pointer_only_licence":0}},{"rank_in_archive_order":11,"model":"MindAct-780M","metrics":{"Element (IoU)":"13.39","Intent Match":"75.87","Overall score":"15.13","Text (F1)":"13.58"},"uses_additional_data":false,"paper_date":"2024-02-08","paper":"/paper/weblinx-real-world-website-navigation-with","paper_url":"https://arxiv.org/abs/2402.05930v2","paper_title":"WebLINX: Real-World Website Navigation with Multi-Turn Dialogue","code":"https://github.com/McGill-NLP/webllama","n_code_links":2,"syntology":{"n_ran":21,"n_unverified":6,"n_samples":27,"n_pointer_only_licence":0}},{"rank_in_archive_order":12,"model":"Flan-T5-250M","metrics":{"Element (IoU)":"14.86","Intent Match":"79.69","Overall score":"14.99","Text (F1)":"9.21"},"uses_additional_data":false,"paper_date":"2024-02-08","paper":"/paper/weblinx-real-world-website-navigation-with","paper_url":"https://arxiv.org/abs/2402.05930v2","paper_title":"WebLINX: Real-World Website Navigation with Multi-Turn Dialogue","code":"https://github.com/McGill-NLP/webllama","n_code_links":2,"syntology":{"n_ran":21,"n_unverified":6,"n_samples":27,"n_pointer_only_licence":0}},{"rank_in_archive_order":13,"model":"MindAct-250M","metrics":{"Element (IoU)":"12.05","Intent Match":"74.25","Overall score":"12.63","Text (F1)":"7.67"},"uses_additional_data":false,"paper_date":"2024-02-08","paper":"/paper/weblinx-real-world-website-navigation-with","paper_url":"https://arxiv.org/abs/2402.05930v2","paper_title":"WebLINX: Real-World Website Navigation with Multi-Turn Dialogue","code":"https://github.com/McGill-NLP/webllama","n_code_links":2,"syntology":{"n_ran":21,"n_unverified":6,"n_samples":27,"n_pointer_only_licence":0}},{"rank_in_archive_order":14,"model":"Pix2Act-282M","metrics":{"Element (IoU)":"6.20","Intent Match":"79.71","Overall score":"12.51","Text (F1)":"16.40"},"uses_additional_data":false,"paper_date":"2024-02-08","paper":"/paper/weblinx-real-world-website-navigation-with","paper_url":"https://arxiv.org/abs/2402.05930v2","paper_title":"WebLINX: Real-World Website Navigation with Multi-Turn Dialogue","code":"https://github.com/McGill-NLP/webllama","n_code_links":2,"syntology":{"n_ran":21,"n_unverified":6,"n_samples":27,"n_pointer_only_licence":0}},{"rank_in_archive_order":15,"model":"GPT-4T (Zero-Shot)","metrics":{"Element (IoU)":"10.85","Intent Match":"41.66","Overall score":"10.72","Text (F1)":"6.75"},"uses_additional_data":false,"paper_date":"2024-02-08","paper":"/paper/weblinx-real-world-website-navigation-with","paper_url":"https://arxiv.org/abs/2402.05930v2","paper_title":"WebLINX: Real-World Website Navigation with Multi-Turn Dialogue","code":"https://github.com/McGill-NLP/webllama","n_code_links":2,"syntology":{"n_ran":21,"n_unverified":6,"n_samples":27,"n_pointer_only_licence":0}},{"rank_in_archive_order":16,"model":"GPT-4V (Zero-Shot)","metrics":{"Element (IoU)":"10.91","Intent Match":"42.36","Overall score":"10.45","Text (F1)":"6.21"},"uses_additional_data":false,"paper_date":"2024-02-08","paper":"/paper/weblinx-real-world-website-navigation-with","paper_url":"https://arxiv.org/abs/2402.05930v2","paper_title":"WebLINX: Real-World Website Navigation with Multi-Turn Dialogue","code":"https://github.com/McGill-NLP/webllama","n_code_links":2,"syntology":{"n_ran":21,"n_unverified":6,"n_samples":27,"n_pointer_only_licence":0}},{"rank_in_archive_order":17,"model":"GPT-3.5T (Zero-Shot)","metrics":{"Element (IoU)":"8.62","Intent Match":"42.77","Overall score":"8.51","Text (F1)":"3.45"},"uses_additional_data":false,"paper_date":"2024-02-08","paper":"/paper/weblinx-real-world-website-navigation-with","paper_url":"https://arxiv.org/abs/2402.05930v2","paper_title":"WebLINX: Real-World Website Navigation with Multi-Turn Dialogue","code":"https://github.com/McGill-NLP/webllama","n_code_links":2,"syntology":{"n_ran":21,"n_unverified":6,"n_samples":27,"n_pointer_only_licence":0}}],"since_archive":{"present":false,"note":"No Syntology-extracted rows are published in this build."},"syntology":{"read_at":"2026-09-24T18:15:14+00:00","claim":"Per row: N of M harvested code samples from that row's paper executed on a synthesized fixture; the other M-N are unverified. Not a reproduction of the row's number; not a correctness claim. n_pointer_only_licence counts samples the site points at rather than redistributes (a licence axis, independent of ran/unverified).","rows_with_graph_line":17,"rows_with_any_sample_ran":17,"distinct_papers_with_graph_line":1,"distinct_papers_with_any_sample_ran":1,"samples_over_distinct_papers":{"n_ran":21,"n_unverified":6,"n_samples":27,"n_pointer_only_licence":0,"note":"each paper (arXiv id) counted once, however many rows it is behind; this is the page-level figure"},"samples_row_weighted":{"n_ran":357,"n_unverified":102,"n_samples":459,"n_pointer_only_licence":0,"note":"row-weighted: a paper behind several rows is counted once per row; inflated relative to samples_over_distinct_papers by design, kept for readers summing the per-row syntology blocks"}}}