{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/is-a-picture-worth-a-thousand-words-delving","title":"Is A Picture Worth A Thousand Words? Delving Into Spatial Reasoning for Vision Language Models","arxiv_id":"2406.14852","date":"2024-06-21","proceeding":null,"authors":["Jiayu Wang","Yifei Ming","Zhenmei Shi","Vibhav Vineet","Xin Wang","Yixuan Li","Neel Joshi"],"abstract":"Large language models (LLMs) and vision-language models (VLMs) have demonstrated remarkable performance across a wide range of tasks and domains. Despite this promise, spatial understanding and reasoning -- a fundamental component of human cognition -- remains under-explored. We propose SpatialEval, a novel benchmark that covers diverse aspects of spatial reasoning such as relationship understanding, navigation, and counting. We conduct a comprehensive evaluation of competitive language and vision-language models. Our findings reveal several counter-intuitive insights that have been overlooked in the literature: (1) Spatial reasoning poses significant challenges where competitive models can fall behind random guessing; (2) Despite additional visual input, VLMs often under-perform compared to their LLM counterparts; (3) When both textual and visual information is available, multi-modal language models become less reliant on visual information if sufficient textual clues are provided. Additionally, we demonstrate that leveraging redundancy between vision and text can significantly enhance model performance. We hope our study will inform the development of multimodal models to improve spatial intelligence and further close the gap with human intelligence.","url_abs":"https://arxiv.org/abs/2406.14852v2","url_pdf":"https://arxiv.org/pdf/2406.14852v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"is-a-picture-worth-a-thousand-words-delving","repo_url":"https://github.com/jiayuww/SpatialEval","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"spatial-reasoning","task_name":"Spatial Reasoning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2406.14852","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.14852"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/jiayuww/SpatialEval","reach":null},{"provenance":"deterministic:regex_extraction","url":"https://github.com/BAAI-DCAI/Bunny","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":1,"ran_draft_wrong":7,"ran_violates":1,"ran_honours":1,"unverified":3},"by_repo_kind":{"official":{"samples":6,"ran":6,"repositories":1},"found_in_text":{"samples":7,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":6,"samples":[{"code_sha256_prefix":"2fd1673b298c086c","entry":"construct_prompt","repo":"BAAI-DCAI/Bunny","repo_kind":"found_in_text","path":"bunny/eval/model_vqa_cmmmu.py","file_url":"https://github.com/BAAI-DCAI/Bunny/blob/HEAD/bunny/eval/model_vqa_cmmmu.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2fd1673b298c086c"}},{"code_sha256_prefix":"e7d1140ae1160f39","entry":"extract_before_is","repo":"jiayuww/SpatialEval","repo_kind":"official","path":"evals/evaluation.py","file_url":"https://github.com/jiayuww/SpatialEval/blob/HEAD/evals/evaluation.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e7d1140ae1160f39"}},{"code_sha256_prefix":"d7410f697a9e72fb","entry":"extract_model_name","repo":"jiayuww/SpatialEval","repo_kind":"official","path":"evals/evaluation.py","file_url":"https://github.com/jiayuww/SpatialEval/blob/HEAD/evals/evaluation.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d7410f697a9e72fb"}},{"code_sha256_prefix":"7dfcd86ef6835d64","entry":"format_bunny_tqa_prompt_hf","repo":"jiayuww/SpatialEval","repo_kind":"official","path":"inference_vlm.py","file_url":"https://github.com/jiayuww/SpatialEval/blob/HEAD/inference_vlm.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7dfcd86ef6835d64"}},{"code_sha256_prefix":"7e7acafc5ee625af","entry":"format_bunny_vqa_prompt_hf","repo":"jiayuww/SpatialEval","repo_kind":"official","path":"inference_vlm.py","file_url":"https://github.com/jiayuww/SpatialEval/blob/HEAD/inference_vlm.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7e7acafc5ee625af"}},{"code_sha256_prefix":"42a46570620cd9fa","entry":"get_chunk","repo":"BAAI-DCAI/Bunny","repo_kind":"found_in_text","path":"bunny/eval/model_vqa.py","file_url":"https://github.com/BAAI-DCAI/Bunny/blob/HEAD/bunny/eval/model_vqa.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"42a46570620cd9fa"}},{"code_sha256_prefix":"bae18947b56f2be1","entry":"is_none","repo":"BAAI-DCAI/Bunny","repo_kind":"found_in_text","path":"bunny/eval/model_vqa_mmbench.py","file_url":"https://github.com/BAAI-DCAI/Bunny/blob/HEAD/bunny/eval/model_vqa_mmbench.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bae18947b56f2be1"}},{"code_sha256_prefix":"85353e219884c7eb","entry":"remove_redundancy","repo":"jiayuww/SpatialEval","repo_kind":"official","path":"evals/evaluation.py","file_url":"https://github.com/jiayuww/SpatialEval/blob/HEAD/evals/evaluation.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"85353e219884c7eb"}},{"code_sha256_prefix":"076c252c52cbb161","entry":"split_list","repo":"BAAI-DCAI/Bunny","repo_kind":"found_in_text","path":"bunny/eval/model_vqa.py","file_url":"https://github.com/BAAI-DCAI/Bunny/blob/HEAD/bunny/eval/model_vqa.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"076c252c52cbb161"}},{"code_sha256_prefix":"e582401378cfcafc","entry":"tokenizer_image_token","repo":"jiayuww/SpatialEval","repo_kind":"official","path":"inference_vlm.py","file_url":"https://github.com/jiayuww/SpatialEval/blob/HEAD/inference_vlm.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e582401378cfcafc"}},{"code_sha256_prefix":"8bc914474691d84a","entry":"load_pretrained_model","repo":"BAAI-DCAI/Bunny","repo_kind":"found_in_text","path":"bunny/model/builder.py","file_url":"https://github.com/BAAI-DCAI/Bunny/blob/HEAD/bunny/model/builder.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8bc914474691d84a"}},{"code_sha256_prefix":"ce93b7e192e1a8b7","entry":"load_yaml","repo":"BAAI-DCAI/Bunny","repo_kind":"found_in_text","path":"bunny/eval/model_vqa_cmmmu.py","file_url":"https://github.com/BAAI-DCAI/Bunny/blob/HEAD/bunny/eval/model_vqa_cmmmu.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ce93b7e192e1a8b7"}},{"code_sha256_prefix":"f3018307f9301da1","entry":"parse_multi_choice_response","repo":"BAAI-DCAI/Bunny","repo_kind":"found_in_text","path":"bunny/eval/model_vqa_mmmu.py","file_url":"https://github.com/BAAI-DCAI/Bunny/blob/HEAD/bunny/eval/model_vqa_mmmu.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f3018307f9301da1"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}