{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2601-08536","title":"DeepResearch Bench II: Diagnosing Deep Research Agents via Rubrics from Expert Report","arxiv_id":"2601.08536","date":"2026-01-13","proceeding":null,"authors":["Ruizhe Li","Mingxuan Du","Benfeng Xu","Chiwei Zhu","Xiaorui Wang","Zhendong Mao"],"abstract":"Deep Research Systems (DRS) aim to help users search the web, synthesize information, and deliver comprehensive investigative reports. However, how to rigorously evaluate these systems remains under-explored. Existing deep-research benchmarks often fall into two failure modes. Some do not adequately test a system's ability to analyze evidence and write coherent reports. Others rely on evaluation criteria that are either overly coarse or directly defined by LLMs (or both), leading to scores that can be biased relative to human experts and are hard to verify or interpret. To address these issues, we introduce Deep Research Bench II, a new benchmark for evaluating DRS-generated reports. It contains 132 grounded research tasks across 22 domains; for each task, a system must produce a long-form research report that is evaluated by a set of 9430 fine-grained binary rubrics in total, covering three dimensions: information recall, analysis, and presentation. All rubrics are derived from carefully selected expert-written investigative articles and are constructed through a four-stage LLM+human pipeline that combines automatic extraction with over 400 human-hours of expert review, ensuring that the criteria are atomic, verifiable, and aligned with human expert judgment. We evaluate several state-of-the-art deep-research systems on Deep Research Bench II and find that even the strongest models satisfy fewer than 50% of the rubrics, revealing a substantial gap between current DRSs and human experts.","url_abs":"https://arxiv.org/abs/2601.08536","url_pdf":"https://arxiv.org/pdf/2601.08536","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2601.08536","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2601.08536"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/imlrz/DeepResearch-Bench-II","reach":null}],"summary":{"ran":1,"ran_draft_wrong":2,"ran_violates":1,"unverified":4},"by_repo_kind":{"found_in_text":{"samples":8,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"842c05e80541a160","entry":"GPTInput","repo":"imlrz/DeepResearch-Bench-II","repo_kind":"found_in_text","path":"run_evaluation.py","file_url":"https://github.com/imlrz/DeepResearch-Bench-II/blob/HEAD/run_evaluation.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"842c05e80541a160"}},{"code_sha256_prefix":"4f70981f79639577","entry":"_table_to_markdown","repo":"imlrz/DeepResearch-Bench-II","repo_kind":"found_in_text","path":"run_evaluation.py","file_url":"https://github.com/imlrz/DeepResearch-Bench-II/blob/HEAD/run_evaluation.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4f70981f79639577"}},{"code_sha256_prefix":"87adb75491de04ea","entry":"parse_model_text","repo":"imlrz/DeepResearch-Bench-II","repo_kind":"found_in_text","path":"run_evaluation.py","file_url":"https://github.com/imlrz/DeepResearch-Bench-II/blob/HEAD/run_evaluation.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"87adb75491de04ea"}},{"code_sha256_prefix":"4e17e678335a4f23","entry":"validate_batch_result","repo":"imlrz/DeepResearch-Bench-II","repo_kind":"found_in_text","path":"run_evaluation.py","file_url":"https://github.com/imlrz/DeepResearch-Bench-II/blob/HEAD/run_evaluation.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4e17e678335a4f23"}},{"code_sha256_prefix":"acbaa1869b77695f","entry":"_extract_docx_content","repo":"imlrz/DeepResearch-Bench-II","repo_kind":"found_in_text","path":"run_evaluation.py","file_url":"https://github.com/imlrz/DeepResearch-Bench-II/blob/HEAD/run_evaluation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"acbaa1869b77695f"}},{"code_sha256_prefix":"b2013c00edd6afd1","entry":"_try_clean_and_load","repo":"imlrz/DeepResearch-Bench-II","repo_kind":"found_in_text","path":"run_evaluation.py","file_url":"https://github.com/imlrz/DeepResearch-Bench-II/blob/HEAD/run_evaluation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b2013c00edd6afd1"}},{"code_sha256_prefix":"3a03bfc5228aa419","entry":"process_one_with_chunking","repo":"imlrz/DeepResearch-Bench-II","repo_kind":"found_in_text","path":"run_evaluation.py","file_url":"https://github.com/imlrz/DeepResearch-Bench-II/blob/HEAD/run_evaluation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3a03bfc5228aa419"}},{"code_sha256_prefix":"dcf3b5766de73df4","entry":"query_rubric_batch","repo":"imlrz/DeepResearch-Bench-II","repo_kind":"found_in_text","path":"run_evaluation.py","file_url":"https://github.com/imlrz/DeepResearch-Bench-II/blob/HEAD/run_evaluation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"dcf3b5766de73df4"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.CL","source":"arxiv_2026.jsonl"},"syntology_extracted_results":null}