{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/benchlmm-benchmarking-cross-style-visual","title":"BenchLMM: Benchmarking Cross-style Visual Capability of Large Multimodal Models","arxiv_id":"2312.02896","date":"2023-12-05","proceeding":null,"authors":["Rizhao Cai","Zirui Song","Dayan Guan","Zhenhao Chen","Xing Luo","Chenyu Yi","Alex Kot"],"abstract":"Large Multimodal Models (LMMs) such as GPT-4V and LLaVA have shown remarkable capabilities in visual reasoning with common image styles. However, their robustness against diverse style shifts, crucial for practical applications, remains largely unexplored. In this paper, we propose a new benchmark, BenchLMM, to assess the robustness of LMMs against three different styles: artistic image style, imaging sensor style, and application style, where each style has five sub-styles. Utilizing BenchLMM, we comprehensively evaluate state-of-the-art LMMs and reveal: 1) LMMs generally suffer performance degradation when working with other styles; 2) An LMM performs better than another model in common style does not guarantee its superior performance in other styles; 3) LMMs' reasoning capability can be enhanced by prompting LMMs to predict the style first, based on which we propose a versatile and training-free method for improving LMMs; 4) An intelligent LMM is expected to interpret the causes of its errors when facing stylistic variations. We hope that our benchmark and analysis can shed new light on developing more intelligent and versatile LMMs.","url_abs":"https://arxiv.org/abs/2312.02896v2","url_pdf":"https://arxiv.org/pdf/2312.02896v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"benchlmm-benchmarking-cross-style-visual","repo_url":"https://github.com/aifeg/benchgpt","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"benchlmm-benchmarking-cross-style-visual","repo_url":"https://github.com/aifeg/benchlmm","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"benchmarking","task_name":"Benchmarking"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"},{"task_slug":"visual-reasoning","task_name":"Visual Reasoning"}],"methods":[],"datasets_introduced":[{"slug":"benchlmm","name":"BenchLMM","full_name":"BenchLMM: Benchmarking Cross-style Visual Capability of Large Multimodal Models"}],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2312.02896","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.02896"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/aifeg/benchgpt","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/aifeg/benchlmm","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":4,"ran_draft_wrong":2},"by_repo_kind":{"official":{"samples":6,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"36c32e209af63b80","entry":"calculate_average_score","repo":"aifeg/benchgpt","repo_kind":"official","path":"evaluate/avg_score.py","file_url":"https://github.com/aifeg/benchgpt/blob/HEAD/evaluate/avg_score.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"36c32e209af63b80"}},{"code_sha256_prefix":"42a46570620cd9fa","entry":"get_chunk","repo":"aifeg/benchgpt","repo_kind":"official","path":"baseline/LLaVA/BenchGPT_LLaVA_model_vqa.py","file_url":"https://github.com/aifeg/benchgpt/blob/HEAD/baseline/LLaVA/BenchGPT_LLaVA_model_vqa.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"42a46570620cd9fa"}},{"code_sha256_prefix":"b9745b5a8436c863","entry":"get_local_image","repo":"aifeg/benchgpt","repo_kind":"official","path":"baseline/InstructBLIP/blip_image_result1.py","file_url":"https://github.com/aifeg/benchgpt/blob/HEAD/baseline/InstructBLIP/blip_image_result1.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b9745b5a8436c863"}},{"code_sha256_prefix":"c37806d66114479a","entry":"load_jsonl","repo":"aifeg/benchgpt","repo_kind":"official","path":"evaluate/gpt_evaluation_script.py","file_url":"https://github.com/aifeg/benchgpt/blob/HEAD/evaluate/gpt_evaluation_script.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c37806d66114479a"}},{"code_sha256_prefix":"4d55e3ac6c2f1ffe","entry":"parse_scores","repo":"aifeg/benchgpt","repo_kind":"official","path":"evaluate/gpt_evaluation_script_Robots_Games.py","file_url":"https://github.com/aifeg/benchgpt/blob/HEAD/evaluate/gpt_evaluation_script_Robots_Games.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4d55e3ac6c2f1ffe"}},{"code_sha256_prefix":"076c252c52cbb161","entry":"split_list","repo":"aifeg/benchgpt","repo_kind":"official","path":"baseline/LLaVA/BenchGPT_LLaVA_model_vqa.py","file_url":"https://github.com/aifeg/benchgpt/blob/HEAD/baseline/LLaVA/BenchGPT_LLaVA_model_vqa.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"076c252c52cbb161"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}