{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/infochartqa-a-benchmark-for-multimodal","title":"InfoChartQA: A Benchmark for Multimodal Question Answering on Infographic Charts","arxiv_id":"2505.19028","date":"2025-05-25","proceeding":null,"authors":["Minzhi Lin","Tianchi Xie","Mengchen Liu","Yilin Ye","Changjian Chen","Shixia Liu"],"abstract":"Understanding infographic charts with design-driven visual elements (e.g., pictograms, icons) requires both visual recognition and reasoning, posing challenges for multimodal large language models (MLLMs). However, existing visual-question answering benchmarks fall short in evaluating these capabilities of MLLMs due to the lack of paired plain charts and visual-element-based questions. To bridge this gap, we introduce InfoChartQA, a benchmark for evaluating MLLMs on infographic chart understanding. It includes 5,642 pairs of infographic and plain charts, each sharing the same underlying data but differing in visual presentations. We further design visual-element-based questions to capture their unique visual designs and communicative intent. Evaluation of 20 MLLMs reveals a substantial performance decline on infographic charts, particularly for visual-element-based questions related to metaphors. The paired infographic and plain charts enable fine-grained error analysis and ablation studies, which highlight new opportunities for advancing MLLMs in infographic chart understanding. We release InfoChartQA at https://github.com/CoolDawnAnt/InfoChartQA.","url_abs":"https://arxiv.org/abs/2505.19028v1","url_pdf":"https://arxiv.org/pdf/2505.19028v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"infochartqa-a-benchmark-for-multimodal","repo_url":"https://github.com/cooldawnant/infochartqa","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"infochartqa-a-benchmark-for-multimodal","repo_url":"https://github.com/chartgalaxy/chartgalaxy","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":null},{"paper_slug":"infochartqa-a-benchmark-for-multimodal","repo_url":"https://github.com/orionbench/orionbench","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"chart-understanding","task_name":"Chart Understanding"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2505.19028","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.19028"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/chartgalaxy/chartgalaxy","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/orionbench/orionbench","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/cooldawnant/infochartqa","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":3,"ran_draft_wrong":3,"unverified":14},"by_repo_kind":{"official":{"samples":12,"ran":0,"repositories":1},"listed":{"samples":8,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"4e2dcdfe578946a0","entry":"check_link","repo":"orionbench/orionbench","repo_kind":"listed","path":"model_evaluation/mmdetection/.dev_scripts/check_links.py","file_url":"https://github.com/orionbench/orionbench/blob/HEAD/model_evaluation/mmdetection/.dev_scripts/check_links.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4e2dcdfe578946a0"}},{"code_sha256_prefix":"cc93fdc812eca8f7","entry":"check_url","repo":"orionbench/orionbench","repo_kind":"listed","path":"model_evaluation/mmdetection/.dev_scripts/check_links.py","file_url":"https://github.com/orionbench/orionbench/blob/HEAD/model_evaluation/mmdetection/.dev_scripts/check_links.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cc93fdc812eca8f7"}},{"code_sha256_prefix":"fb35a4fb403a10b9","entry":"fix_list_format","repo":"orionbench/orionbench","repo_kind":"listed","path":"grounded_CoT/evaluate_predictions.py","file_url":"https://github.com/orionbench/orionbench/blob/HEAD/grounded_CoT/evaluate_predictions.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fb35a4fb403a10b9"}},{"code_sha256_prefix":"36667e7b107acaec","entry":"load_predictions","repo":"orionbench/orionbench","repo_kind":"listed","path":"grounded_CoT/evaluate_predictions.py","file_url":"https://github.com/orionbench/orionbench/blob/HEAD/grounded_CoT/evaluate_predictions.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"36667e7b107acaec"}},{"code_sha256_prefix":"af8755d4fa414194","entry":"parse_to_list","repo":"orionbench/orionbench","repo_kind":"listed","path":"grounded_CoT/evaluate_predictions.py","file_url":"https://github.com/orionbench/orionbench/blob/HEAD/grounded_CoT/evaluate_predictions.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"af8755d4fa414194"}},{"code_sha256_prefix":"3175ca3b1ec8c7b9","entry":"process_model_info","repo":"orionbench/orionbench","repo_kind":"listed","path":"model_evaluation/mmdetection/.dev_scripts/convert_test_benchmark_script.py","file_url":"https://github.com/orionbench/orionbench/blob/HEAD/model_evaluation/mmdetection/.dev_scripts/convert_test_benchmark_script.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3175ca3b1ec8c7b9"}},{"code_sha256_prefix":"4c8cc106f532a2e5","entry":"build_question","repo":"cooldawnant/infochartqa","repo_kind":"official","path":"eval/example.py","file_url":"https://github.com/cooldawnant/infochartqa/blob/HEAD/eval/example.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4c8cc106f532a2e5"}},{"code_sha256_prefix":"fd08138d4bde9cfa","entry":"check_path","repo":"orionbench/orionbench","repo_kind":"listed","path":"model_evaluation/mmdetection/.dev_scripts/check_links.py","file_url":"https://github.com/orionbench/orionbench/blob/HEAD/model_evaluation/mmdetection/.dev_scripts/check_links.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fd08138d4bde9cfa"}},{"code_sha256_prefix":"fdef7359919a4918","entry":"determine_gpus","repo":"orionbench/orionbench","repo_kind":"listed","path":"model_evaluation/mmdetection/.dev_scripts/convert_train_benchmark_script.py","file_url":"https://github.com/orionbench/orionbench/blob/HEAD/model_evaluation/mmdetection/.dev_scripts/convert_train_benchmark_script.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fdef7359919a4918"}},{"code_sha256_prefix":"1b6f238ba93da715","entry":"extract_answers_from_file","repo":"cooldawnant/infochartqa","repo_kind":"official","path":"eval/compare_sequence.py","file_url":"https://github.com/cooldawnant/infochartqa/blob/HEAD/eval/compare_sequence.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1b6f238ba93da715"}},{"code_sha256_prefix":"28a88df4a1890f0c","entry":"extract_numbers_keep_order","repo":"cooldawnant/infochartqa","repo_kind":"official","path":"eval/compare_value.py","file_url":"https://github.com/cooldawnant/infochartqa/blob/HEAD/eval/compare_value.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"28a88df4a1890f0c"}},{"code_sha256_prefix":"ad68de6db9bd4905","entry":"fuzzy_match","repo":"cooldawnant/infochartqa","repo_kind":"official","path":"eval/compare_sequence.py","file_url":"https://github.com/cooldawnant/infochartqa/blob/HEAD/eval/compare_sequence.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ad68de6db9bd4905"}},{"code_sha256_prefix":"dbdd053033f11e57","entry":"fuzzy_string_match","repo":"cooldawnant/infochartqa","repo_kind":"official","path":"eval/compare_str.py","file_url":"https://github.com/cooldawnant/infochartqa/blob/HEAD/eval/compare_str.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"dbdd053033f11e57"}},{"code_sha256_prefix":"29bff60fbb90eec9","entry":"get_last_number","repo":"cooldawnant/infochartqa","repo_kind":"official","path":"eval/compare_value.py","file_url":"https://github.com/cooldawnant/infochartqa/blob/HEAD/eval/compare_value.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"29bff60fbb90eec9"}},{"code_sha256_prefix":"b20b7cb539fda947","entry":"has_more_digits_than_other_chars","repo":"cooldawnant/infochartqa","repo_kind":"official","path":"eval/checker.py","file_url":"https://github.com/cooldawnant/infochartqa/blob/HEAD/eval/checker.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b20b7cb539fda947"}},{"code_sha256_prefix":"a51282a3ec6078f1","entry":"is_sequence_valid","repo":"cooldawnant/infochartqa","repo_kind":"official","path":"eval/compare_sequence.py","file_url":"https://github.com/cooldawnant/infochartqa/blob/HEAD/eval/compare_sequence.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a51282a3ec6078f1"}},{"code_sha256_prefix":"3c2a6db01c13dd65","entry":"is_string_match","repo":"cooldawnant/infochartqa","repo_kind":"official","path":"eval/compare_str.py","file_url":"https://github.com/cooldawnant/infochartqa/blob/HEAD/eval/compare_str.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3c2a6db01c13dd65"}},{"code_sha256_prefix":"fcf8f6cd8a2cd9b4","entry":"multiple_choice_checker","repo":"cooldawnant/infochartqa","repo_kind":"official","path":"eval/compare_multiple.py","file_url":"https://github.com/cooldawnant/infochartqa/blob/HEAD/eval/compare_multiple.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fcf8f6cd8a2cd9b4"}},{"code_sha256_prefix":"1e56ccd851128c34","entry":"only_digits_and_commas","repo":"cooldawnant/infochartqa","repo_kind":"official","path":"eval/compare_value.py","file_url":"https://github.com/cooldawnant/infochartqa/blob/HEAD/eval/compare_value.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1e56ccd851128c34"}},{"code_sha256_prefix":"d86d4c52f0d2788b","entry":"parse_multi_choice_response","repo":"cooldawnant/infochartqa","repo_kind":"official","path":"eval/compare_multiple.py","file_url":"https://github.com/cooldawnant/infochartqa/blob/HEAD/eval/compare_multiple.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d86d4c52f0d2788b"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}