{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/mirage-a-benchmark-for-multimodal-information","title":"MIRAGE: A Benchmark for Multimodal Information-Seeking and Reasoning in Agricultural Expert-Guided Conversations","arxiv_id":"2506.20100","date":"2025-06-25","proceeding":null,"authors":["Vardhan Dongre","Chi Gui","Shubham Garg","Hooshang Nayyeri","Gokhan Tur","Dilek Hakkani-Tür","Vikram S. Adve"],"abstract":"We introduce MIRAGE, a new benchmark for multimodal expert-level reasoning and decision-making in consultative interaction settings. Designed for the agriculture domain, MIRAGE captures the full complexity of expert consultations by combining natural user queries, expert-authored responses, and image-based context, offering a high-fidelity benchmark for evaluating models on grounded reasoning, clarification strategies, and long-form generation in a real-world, knowledge-intensive domain. Grounded in over 35,000 real user-expert interactions and curated through a carefully designed multi-step pipeline, MIRAGE spans diverse crop health, pest diagnosis, and crop management scenarios. The benchmark includes more than 7,000 unique biological entities, covering plant species, pests, and diseases, making it one of the most taxonomically diverse benchmarks available for vision-language models, grounded in the real world. Unlike existing benchmarks that rely on well-specified user inputs and closed-set taxonomies, MIRAGE features underspecified, context-rich scenarios with open-world settings, requiring models to infer latent knowledge gaps, handle rare entities, and either proactively guide the interaction or respond. Project Page: https://mirage-benchmark.github.io","url_abs":"https://arxiv.org/abs/2506.20100v1","url_pdf":"https://arxiv.org/pdf/2506.20100v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"mirage-a-benchmark-for-multimodal-information","repo_url":"https://github.com/mirage-benchmark/mirage-benchmark","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}}],"tasks":[{"task_slug":"world-knowledge","task_name":"World Knowledge"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2506.20100","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2506.20100"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mirage-benchmark/mirage-benchmark","reach":{"status":"ok"}}],"summary":{"ran_draft_wrong":1,"unverified":17},"by_repo_kind":{"official":{"samples":18,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":18,"samples":[{"code_sha256_prefix":"f41cb1a19b154297","entry":"encode_image","repo":"mirage-benchmark/mirage-benchmark","repo_kind":"official","path":"MMST/chat_models/Client.py","file_url":"https://github.com/mirage-benchmark/mirage-benchmark/blob/HEAD/MMST/chat_models/Client.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f41cb1a19b154297"}},{"code_sha256_prefix":"bf40ff2b135468ac","entry":"batch_generate","repo":"mirage-benchmark/mirage-benchmark","repo_kind":"official","path":"MMMT/src/llm_agents.py","file_url":"https://github.com/mirage-benchmark/mirage-benchmark/blob/HEAD/MMMT/src/llm_agents.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bf40ff2b135468ac"}},{"code_sha256_prefix":"599869e892bfa6cf","entry":"build_dialog_context","repo":"mirage-benchmark/mirage-benchmark","repo_kind":"official","path":"MMMT/generate_tasks.py","file_url":"https://github.com/mirage-benchmark/mirage-benchmark/blob/HEAD/MMMT/generate_tasks.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"599869e892bfa6cf"}},{"code_sha256_prefix":"83a5d5145699b180","entry":"build_eval_prompt","repo":"mirage-benchmark/mirage-benchmark","repo_kind":"official","path":"MMMT/src/prompt_builder.py","file_url":"https://github.com/mirage-benchmark/mirage-benchmark/blob/HEAD/MMMT/src/prompt_builder.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"83a5d5145699b180"}},{"code_sha256_prefix":"af46eb3922de5305","entry":"build_eval_prompt_decomp","repo":"mirage-benchmark/mirage-benchmark","repo_kind":"official","path":"MMMT/src/prompt_builder.py","file_url":"https://github.com/mirage-benchmark/mirage-benchmark/blob/HEAD/MMMT/src/prompt_builder.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"af46eb3922de5305"}},{"code_sha256_prefix":"73f970619d92b562","entry":"build_prompt","repo":"mirage-benchmark/mirage-benchmark","repo_kind":"official","path":"MMMT/src/prompt_builder.py","file_url":"https://github.com/mirage-benchmark/mirage-benchmark/blob/HEAD/MMMT/src/prompt_builder.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"73f970619d92b562"}},{"code_sha256_prefix":"a8aa34413805cff7","entry":"detect_speaker","repo":"mirage-benchmark/mirage-benchmark","repo_kind":"official","path":"MMMT/generate_tasks.py","file_url":"https://github.com/mirage-benchmark/mirage-benchmark/blob/HEAD/MMMT/generate_tasks.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a8aa34413805cff7"}},{"code_sha256_prefix":"9593836b0ce2c31a","entry":"extract_json","repo":"mirage-benchmark/mirage-benchmark","repo_kind":"official","path":"MMMT/src/clarification_generator.py","file_url":"https://github.com/mirage-benchmark/mirage-benchmark/blob/HEAD/MMMT/src/clarification_generator.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9593836b0ce2c31a"}},{"code_sha256_prefix":"da20f6bab2d17876","entry":"extract_predictions_from_log","repo":"mirage-benchmark/mirage-benchmark","repo_kind":"official","path":"MMMT/evaluate/evalaute_reason.py","file_url":"https://github.com/mirage-benchmark/mirage-benchmark/blob/HEAD/MMMT/evaluate/evalaute_reason.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"da20f6bab2d17876"}},{"code_sha256_prefix":"e5a0abc6d1e181f7","entry":"extract_predictions_from_log","repo":"mirage-benchmark/mirage-benchmark","repo_kind":"official","path":"MMMT/evaluate/evalaute_zs.py","file_url":"https://github.com/mirage-benchmark/mirage-benchmark/blob/HEAD/MMMT/evaluate/evalaute_zs.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e5a0abc6d1e181f7"}},{"code_sha256_prefix":"e2774d672afc0261","entry":"get_image_captions","repo":"mirage-benchmark/mirage-benchmark","repo_kind":"official","path":"MMMT/src/data_utils.py","file_url":"https://github.com/mirage-benchmark/mirage-benchmark/blob/HEAD/MMMT/src/data_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e2774d672afc0261"}},{"code_sha256_prefix":"8bedfe018838a85a","entry":"get_llm_client","repo":"mirage-benchmark/mirage-benchmark","repo_kind":"official","path":"MMMT/src/llm_agents.py","file_url":"https://github.com/mirage-benchmark/mirage-benchmark/blob/HEAD/MMMT/src/llm_agents.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8bedfe018838a85a"}},{"code_sha256_prefix":"b1ee0080cf9e0749","entry":"load_jsonl_or_json","repo":"mirage-benchmark/mirage-benchmark","repo_kind":"official","path":"MMMT/evaluate/evalaute_reason.py","file_url":"https://github.com/mirage-benchmark/mirage-benchmark/blob/HEAD/MMMT/evaluate/evalaute_reason.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b1ee0080cf9e0749"}},{"code_sha256_prefix":"0ce8598d5845d362","entry":"load_jsonl_or_json","repo":"mirage-benchmark/mirage-benchmark","repo_kind":"official","path":"MMMT/evaluate/evalaute_zs.py","file_url":"https://github.com/mirage-benchmark/mirage-benchmark/blob/HEAD/MMMT/evaluate/evalaute_zs.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0ce8598d5845d362"}},{"code_sha256_prefix":"007e1998bace6aed","entry":"safe_json_parse","repo":"mirage-benchmark/mirage-benchmark","repo_kind":"official","path":"MMMT/evaluate/evalaute_reason.py","file_url":"https://github.com/mirage-benchmark/mirage-benchmark/blob/HEAD/MMMT/evaluate/evalaute_reason.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"007e1998bace6aed"}},{"code_sha256_prefix":"27ae3770ecd5207a","entry":"stream_generate","repo":"mirage-benchmark/mirage-benchmark","repo_kind":"official","path":"MMMT/src/llm_agents.py","file_url":"https://github.com/mirage-benchmark/mirage-benchmark/blob/HEAD/MMMT/src/llm_agents.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"27ae3770ecd5207a"}},{"code_sha256_prefix":"5de8890f8187b489","entry":"truncate_dialog","repo":"mirage-benchmark/mirage-benchmark","repo_kind":"official","path":"MMMT/src/data_utils.py","file_url":"https://github.com/mirage-benchmark/mirage-benchmark/blob/HEAD/MMMT/src/data_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5de8890f8187b489"}},{"code_sha256_prefix":"aa4fea1d2088ded2","entry":"validate_decision","repo":"mirage-benchmark/mirage-benchmark","repo_kind":"official","path":"MMMT/src/validator.py","file_url":"https://github.com/mirage-benchmark/mirage-benchmark/blob/HEAD/MMMT/src/validator.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"aa4fea1d2088ded2"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}