{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/idat-a-multi-modal-dataset-and-toolkit-for","title":"IDAT: A Multi-Modal Dataset and Toolkit for Building and Evaluating Interactive Task-Solving Agents","arxiv_id":"2407.08898","date":"2024-07-12","proceeding":null,"authors":["Shrestha Mohanty","Negar Arabzadeh","Andrea Tupini","Yuxuan Sun","Alexey Skrynnik","Artem Zholus","Marc-Alexandre Côté","Julia Kiseleva"],"abstract":"Seamless interaction between AI agents and humans using natural language remains a key goal in AI research. This paper addresses the challenges of developing interactive agents capable of understanding and executing grounded natural language instructions through the IGLU competition at NeurIPS. Despite advancements, challenges such as a scarcity of appropriate datasets and the need for effective evaluation platforms persist. We introduce a scalable data collection tool for gathering interactive grounded language instructions within a Minecraft-like environment, resulting in a Multi-Modal dataset with around 9,000 utterances and over 1,000 clarification questions. Additionally, we present a Human-in-the-Loop interactive evaluation platform for qualitative analysis and comparison of agent performance through multi-turn communication with human annotators. We offer to the community these assets referred to as IDAT (IGLU Dataset And Toolkit) which aim to advance the development of intelligent, interactive AI agents and provide essential resources for further research.","url_abs":"https://arxiv.org/abs/2407.08898v1","url_pdf":"https://arxiv.org/pdf/2407.08898v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"idat-a-multi-modal-dataset-and-toolkit-for","repo_url":"https://github.com/microsoft/iglu-datasets","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"none","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"minecraft","task_name":"Minecraft"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2407.08898","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.08898"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/microsoft/iglu-datasets","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":3},"by_repo_kind":{"official":{"samples":3,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"f78469d36f61e7f5","entry":"download_azure","repo":"microsoft/iglu-datasets","repo_kind":"official","path":"iglu_dataset_scripts/download.py","file_url":"https://github.com/microsoft/iglu-datasets/blob/HEAD/iglu_dataset_scripts/download.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f78469d36f61e7f5"}},{"code_sha256_prefix":"f892aa901fa1b4f6","entry":"fix_log","repo":"microsoft/iglu-datasets","repo_kind":"official","path":"iglu_dataset_scripts/common.py","file_url":"https://github.com/microsoft/iglu-datasets/blob/HEAD/iglu_dataset_scripts/common.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f892aa901fa1b4f6"}},{"code_sha256_prefix":"f0623ec066754b01","entry":"fix_xyz","repo":"microsoft/iglu-datasets","repo_kind":"official","path":"iglu_dataset_scripts/common.py","file_url":"https://github.com/microsoft/iglu-datasets/blob/HEAD/iglu_dataset_scripts/common.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f0623ec066754b01"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}