{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/unsolvable-problem-detection-evaluating","title":"Unsolvable Problem Detection: Evaluating Trustworthiness of Vision Language Models","arxiv_id":"2403.20331","date":"2024-03-29","proceeding":null,"authors":["Atsuyuki Miyai","Jingkang Yang","Jingyang Zhang","Yifei Ming","Qing Yu","Go Irie","Yixuan Li","Hai Li","Ziwei Liu","Kiyoharu Aizawa"],"abstract":"This paper introduces a novel and significant challenge for Vision Language Models (VLMs), termed Unsolvable Problem Detection (UPD). UPD examines the VLM's ability to withhold answers when faced with unsolvable problems in the context of Visual Question Answering (VQA) tasks. UPD encompasses three distinct settings: Absent Answer Detection (AAD), Incompatible Answer Set Detection (IASD), and Incompatible Visual Question Detection (IVQD). To deeply investigate the UPD problem, extensive experiments indicate that most VLMs, including GPT-4V and LLaVA-Next-34B, struggle with our benchmarks to varying extents, highlighting significant room for the improvements. To address UPD, we explore both training-free and training-based solutions, offering new insights into their effectiveness and limitations. We hope our insights, together with future efforts within the proposed UPD settings, will enhance the broader understanding and development of more practical and reliable VLMs.","url_abs":"https://arxiv.org/abs/2403.20331v1","url_pdf":"https://arxiv.org/pdf/2403.20331v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"unsolvable-problem-detection-evaluating","repo_url":"https://github.com/atsumiyai/upd","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"},{"task_slug":"visual-question-answering","task_name":"Visual Question Answering (VQA)"}],"methods":[{"method_slug":"set","method_name":"SET"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2403.20331","atlas_url":"https://app.syntology.ai/?focus=2403.20331","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.20331"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/atsumiyai/upd","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":7,"unverified":2},"by_repo_kind":{"official":{"samples":9,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"e7c04c2111338866","entry":"build_option_str_w_gt","repo":"atsumiyai/upd","repo_kind":"official","path":"automatic_eval/calculate_scores.py","file_url":"https://github.com/atsumiyai/upd/blob/HEAD/automatic_eval/calculate_scores.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e7c04c2111338866"}},{"code_sha256_prefix":"189c31121c050c08","entry":"eval_result_dual","repo":"atsumiyai/upd","repo_kind":"official","path":"automatic_eval/utils/common_utils.py","file_url":"https://github.com/atsumiyai/upd/blob/HEAD/automatic_eval/utils/common_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"189c31121c050c08"}},{"code_sha256_prefix":"6368891e7477c9bb","entry":"extract_options","repo":"atsumiyai/upd","repo_kind":"official","path":"automatic_eval/calculate_scores.py","file_url":"https://github.com/atsumiyai/upd/blob/HEAD/automatic_eval/calculate_scores.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6368891e7477c9bb"}},{"code_sha256_prefix":"6c59a594c6d59480","entry":"get_options","repo":"atsumiyai/upd","repo_kind":"official","path":"vlms/cogvlm/cogvlm_vqa_updbench.py","file_url":"https://github.com/atsumiyai/upd/blob/HEAD/vlms/cogvlm/cogvlm_vqa_updbench.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6c59a594c6d59480"}},{"code_sha256_prefix":"0e7cf115754e1730","entry":"is_none","repo":"atsumiyai/upd","repo_kind":"official","path":"automatic_eval/calculate_scores.py","file_url":"https://github.com/atsumiyai/upd/blob/HEAD/automatic_eval/calculate_scores.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0e7cf115754e1730"}},{"code_sha256_prefix":"c3ee9d07c900dd55","entry":"load_image_from_base64","repo":"atsumiyai/upd","repo_kind":"official","path":"vlms/cogvlm/cogvlm_vqa_updbench.py","file_url":"https://github.com/atsumiyai/upd/blob/HEAD/vlms/cogvlm/cogvlm_vqa_updbench.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c3ee9d07c900dd55"}},{"code_sha256_prefix":"9637be07a8656049","entry":"safe_format","repo":"atsumiyai/upd","repo_kind":"official","path":"automatic_eval/utils/prompt.py","file_url":"https://github.com/atsumiyai/upd/blob/HEAD/automatic_eval/utils/prompt.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9637be07a8656049"}},{"code_sha256_prefix":"655baabbfd938995","entry":"dump","repo":"atsumiyai/upd","repo_kind":"official","path":"automatic_eval/utils/common_utils.py","file_url":"https://github.com/atsumiyai/upd/blob/HEAD/automatic_eval/utils/common_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"655baabbfd938995"}},{"code_sha256_prefix":"c80c68ae8a315a3f","entry":"load","repo":"atsumiyai/upd","repo_kind":"official","path":"automatic_eval/utils/common_utils.py","file_url":"https://github.com/atsumiyai/upd/blob/HEAD/automatic_eval/utils/common_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c80c68ae8a315a3f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}