{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/omnidocbench-benchmarking-diverse-pdf","title":"OmniDocBench: Benchmarking Diverse PDF Document Parsing with Comprehensive Annotations","arxiv_id":"2412.07626","date":"2024-12-10","proceeding":"CVPR 2025 1","authors":["Linke Ouyang","Yuan Qu","Hongbin Zhou","Jiawei Zhu","Rui Zhang","Qunshu Lin","Bin Wang","Zhiyuan Zhao","Man Jiang","Xiaomeng Zhao","Jin Shi","Fan Wu","Pei Chu","Minghao Liu","Zhenxiang Li","Chao Xu","Bo Zhang","Botian Shi","Zhongying Tu","Conghui He"],"abstract":"Document content extraction is a critical task in computer vision, underpinning the data needs of large language models (LLMs) and retrieval-augmented generation (RAG) systems. Despite recent progress, current document parsing methods have not been fairly and comprehensively evaluated due to the narrow coverage of document types and the simplified, unrealistic evaluation procedures in existing benchmarks. To address these gaps, we introduce OmniDocBench, a novel benchmark featuring high-quality annotations across nine document sources, including academic papers, textbooks, and more challenging cases such as handwritten notes and densely typeset newspapers. OmniDocBench supports flexible, multi-level evaluations--ranging from an end-to-end assessment to the task-specific and attribute--based analysis using 19 layout categories and 15 attribute labels. We conduct a thorough evaluation of both pipeline-based methods and end-to-end vision-language models, revealing their strengths and weaknesses across different document types. OmniDocBench sets a new standard for the fair, diverse, and fine-grained evaluation in document parsing. Dataset and code are available at https://github.com/opendatalab/OmniDocBench.","url_abs":"https://arxiv.org/abs/2412.07626v2","url_pdf":"https://arxiv.org/pdf/2412.07626v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"omnidocbench-benchmarking-diverse-pdf","repo_url":"https://github.com/opendatalab/OmniDocBench","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"paddle","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"attribute","task_name":"Attribute"},{"task_slug":"benchmarking","task_name":"Benchmarking"},{"task_slug":"diversity","task_name":"Diversity"},{"task_slug":"rag","task_name":"RAG"},{"task_slug":"retrieval-augmented-generation","task_name":"Retrieval-augmented Generation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2412.07626","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2412.07626"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/opendatalab/OmniDocBench","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"unverified":15},"by_repo_kind":{"official":{"samples":15,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"ca38ddb21ef8ef71","entry":"build_metric_page_denominators","repo":"opendatalab/OmniDocBench","repo_kind":"official","path":"src/runtime/eval_report.py","file_url":"https://github.com/opendatalab/OmniDocBench/blob/HEAD/src/runtime/eval_report.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ca38ddb21ef8ef71"}},{"code_sha256_prefix":"89eb6100e7a1d37a","entry":"build_notebook_metric_summary","repo":"opendatalab/OmniDocBench","repo_kind":"official","path":"src/runtime/eval_report.py","file_url":"https://github.com/opendatalab/OmniDocBench/blob/HEAD/src/runtime/eval_report.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"89eb6100e7a1d37a"}},{"code_sha256_prefix":"cb4301dfcac9b34a","entry":"build_save_name","repo":"opendatalab/OmniDocBench","repo_kind":"official","path":"src/core/pipeline.py","file_url":"https://github.com/opendatalab/OmniDocBench/blob/HEAD/src/core/pipeline.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cb4301dfcac9b34a"}},{"code_sha256_prefix":"11c0d80bcce3305a","entry":"build_stage_execution_summary","repo":"opendatalab/OmniDocBench","repo_kind":"official","path":"src/runtime/eval_report.py","file_url":"https://github.com/opendatalab/OmniDocBench/blob/HEAD/src/runtime/eval_report.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"11c0d80bcce3305a"}},{"code_sha256_prefix":"361ddd5fb2f646c2","entry":"compute_edit_distance_matrix_new","repo":"opendatalab/OmniDocBench","repo_kind":"official","path":"src/core/matching/match.py","file_url":"https://github.com/opendatalab/OmniDocBench/blob/HEAD/src/core/matching/match.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"361ddd5fb2f646c2"}},{"code_sha256_prefix":"3520da40faf14d34","entry":"get_full_labels_results","repo":"opendatalab/OmniDocBench","repo_kind":"official","path":"src/core/metrics.py","file_url":"https://github.com/opendatalab/OmniDocBench/blob/HEAD/src/core/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3520da40faf14d34"}},{"code_sha256_prefix":"f8346cfe1dd760da","entry":"get_groups","repo":"opendatalab/OmniDocBench","repo_kind":"official","path":"src/metrics/cal_metric.py","file_url":"https://github.com/opendatalab/OmniDocBench/blob/HEAD/src/metrics/cal_metric.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f8346cfe1dd760da"}},{"code_sha256_prefix":"10cc9e5029893cca","entry":"get_page_split","repo":"opendatalab/OmniDocBench","repo_kind":"official","path":"src/core/metrics.py","file_url":"https://github.com/opendatalab/OmniDocBench/blob/HEAD/src/core/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"10cc9e5029893cca"}},{"code_sha256_prefix":"a287771e93998194","entry":"get_pred_category_type","repo":"opendatalab/OmniDocBench","repo_kind":"official","path":"src/core/matching/match.py","file_url":"https://github.com/opendatalab/OmniDocBench/blob/HEAD/src/core/matching/match.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a287771e93998194"}},{"code_sha256_prefix":"643b624f3dbd2451","entry":"load_config","repo":"opendatalab/OmniDocBench","repo_kind":"official","path":"src/core/pipeline.py","file_url":"https://github.com/opendatalab/OmniDocBench/blob/HEAD/src/core/pipeline.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"643b624f3dbd2451"}},{"code_sha256_prefix":"462931ce8ed635f0","entry":"process_args","repo":"opendatalab/OmniDocBench","repo_kind":"official","path":"src/core/pipeline.py","file_url":"https://github.com/opendatalab/OmniDocBench/blob/HEAD/src/core/pipeline.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"462931ce8ed635f0"}},{"code_sha256_prefix":"cbee43d2ecc4107f","entry":"resolve_cdm_workers","repo":"opendatalab/OmniDocBench","repo_kind":"official","path":"src/runtime/concurrency.py","file_url":"https://github.com/opendatalab/OmniDocBench/blob/HEAD/src/runtime/concurrency.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cbee43d2ecc4107f"}},{"code_sha256_prefix":"919efacf435b6e47","entry":"resolve_match_workers","repo":"opendatalab/OmniDocBench","repo_kind":"official","path":"src/runtime/concurrency.py","file_url":"https://github.com/opendatalab/OmniDocBench/blob/HEAD/src/runtime/concurrency.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"919efacf435b6e47"}},{"code_sha256_prefix":"17f3204eb9743817","entry":"resolve_teds_workers","repo":"opendatalab/OmniDocBench","repo_kind":"official","path":"src/runtime/concurrency.py","file_url":"https://github.com/opendatalab/OmniDocBench/blob/HEAD/src/runtime/concurrency.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"17f3204eb9743817"}},{"code_sha256_prefix":"73ff55a713b31dd3","entry":"sort_nested_dict","repo":"opendatalab/OmniDocBench","repo_kind":"official","path":"src/core/metrics.py","file_url":"https://github.com/opendatalab/OmniDocBench/blob/HEAD/src/core/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"73ff55a713b31dd3"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}