{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/good-at-captioning-bad-at-counting","title":"Good at captioning, bad at counting: Benchmarking GPT-4V on Earth observation data","arxiv_id":"2401.17600","date":"2024-01-31","proceeding":null,"authors":["Chenhui Zhang","Sherrie Wang"],"abstract":"Large Vision-Language Models (VLMs) have demonstrated impressive performance on complex tasks involving visual input with natural language instructions. However, it remains unclear to what extent capabilities on natural images transfer to Earth observation (EO) data, which are predominantly satellite and aerial images less common in VLM training data. In this work, we propose a comprehensive benchmark to gauge the progress of VLMs toward being useful tools for EO data by assessing their abilities on scene understanding, localization and counting, and change detection tasks. Motivated by real-world applications, our benchmark includes scenarios like urban monitoring, disaster relief, land use, and conservation. We discover that, although state-of-the-art VLMs like GPT-4V possess extensive world knowledge that leads to strong performance on open-ended tasks like location understanding and image captioning, their poor spatial reasoning limits usefulness on object localization and counting tasks. Our benchmark will be made publicly available at https://vleo.danielz.ch/ and on Hugging Face at https://huggingface.co/collections/mit-ei/vleo-benchmark-datasets-65b789b0466555489cce0d70 for easy model evaluation.","url_abs":"https://arxiv.org/abs/2401.17600v1","url_pdf":"https://arxiv.org/pdf/2401.17600v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"good-at-captioning-bad-at-counting","repo_url":"https://github.com/Earth-Intelligence-Lab/vleo-bench","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"none","reach":null}],"tasks":[{"task_slug":"benchmarking","task_name":"Benchmarking"},{"task_slug":"change-detection","task_name":"Change Detection"},{"task_slug":"earth-observation","task_name":"Earth Observation"},{"task_slug":"image-captioning","task_name":"Image Captioning"},{"task_slug":"object-localization","task_name":"Object Localization"},{"task_slug":"scene-understanding","task_name":"Scene Understanding"},{"task_slug":"spatial-reasoning","task_name":"Spatial Reasoning"},{"task_slug":"world-knowledge","task_name":"World Knowledge"}],"methods":[],"datasets_introduced":[{"slug":"aerial-landmarks-recognition-dataset","name":"Aerial Landmarks Recognition Dataset","full_name":""}],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2401.17600","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.17600"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Earth-Intelligence-Lab/vleo-bench","reach":null}],"summary":{"ran_draft_wrong":2},"by_repo_kind":{"official":{"samples":1,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":2,"samples":[{"code_sha256_prefix":"a79590e3a6c83079","entry":"base64_image_to_bytes","repo":"Earth-Intelligence-Lab/vleo-bench","repo_kind":"official","path":"src/utils/gpt4v_chat.py","file_url":"https://github.com/Earth-Intelligence-Lab/vleo-bench/blob/HEAD/src/utils/gpt4v_chat.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a79590e3a6c83079"}},{"code_sha256_prefix":"f41cb1a19b154297","entry":"encode_image","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"f41cb1a19b154297"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}