{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/conme-rethinking-evaluation-of-compositional","title":"ConMe: Rethinking Evaluation of Compositional Reasoning for Modern VLMs","arxiv_id":"2406.08164","date":"2024-06-12","proceeding":null,"authors":["Irene Huang","Wei Lin","M. Jehanzeb Mirza","Jacob A. Hansen","Sivan Doveh","Victor Ion Butoi","Roei Herzig","Assaf Arbelle","Hilde Kuehne","Trevor Darrell","Chuang Gan","Aude Oliva","Rogerio Feris","Leonid Karlinsky"],"abstract":"Compositional Reasoning (CR) entails grasping the significance of attributes, relations, and word order. Recent Vision-Language Models (VLMs), comprising a visual encoder and a Large Language Model (LLM) decoder, have demonstrated remarkable proficiency in such reasoning tasks. This prompts a crucial question: have VLMs effectively tackled the CR challenge? We conjecture that existing CR benchmarks may not adequately push the boundaries of modern VLMs due to the reliance on an LLM-only negative text generation pipeline. Consequently, the negatives produced either appear as outliers from the natural language distribution learned by VLMs' LLM decoders or as improbable within the corresponding image context. To address these limitations, we introduce ConMe -- a compositional reasoning benchmark and a novel data generation pipeline leveraging VLMs to produce `hard CR Q&A'. Through a new concept of VLMs conversing with each other to collaboratively expose their weaknesses, our pipeline autonomously generates, evaluates, and selects challenging compositional reasoning questions, establishing a robust CR benchmark, also subsequently validated manually. Our benchmark provokes a noteworthy, up to 33%, decrease in CR performance compared to preceding benchmarks, reinstating the CR challenge even for state-of-the-art VLMs.","url_abs":"https://arxiv.org/abs/2406.08164v3","url_pdf":"https://arxiv.org/pdf/2406.08164v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"conme-rethinking-evaluation-of-compositional","repo_url":"https://github.com/jmiemirza/conme","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"decoder","task_name":"Decoder"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"large-language-model","task_name":"Large Language Model"},{"task_slug":"text-generation","task_name":"Text Generation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2406.08164","atlas_url":"https://app.syntology.ai/?focus=2406.08164","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.08164"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jmiemirza/conme","reach":{"status":"ok"}}],"summary":{"ran":11,"unverified":1},"by_repo_kind":{"official":{"samples":12,"ran":11,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":12,"samples":[{"code_sha256_prefix":"52f205326f1875c9","entry":"check_contradict","repo":"jmiemirza/conme","repo_kind":"official","path":"prompt_negatives.py","file_url":"https://github.com/jmiemirza/conme/blob/HEAD/prompt_negatives.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"52f205326f1875c9"}},{"code_sha256_prefix":"acfe0b9350deeb90","entry":"check_contradict","repo":"jmiemirza/conme","repo_kind":"official","path":"prompt_negatives/classify_contradictions.py","file_url":"https://github.com/jmiemirza/conme/blob/HEAD/prompt_negatives/classify_contradictions.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"acfe0b9350deeb90"}},{"code_sha256_prefix":"88e6f2dd71a42f60","entry":"check_contradict","repo":"jmiemirza/conme","repo_kind":"official","path":"prompt_negatives/classify_llava_mix.py","file_url":"https://github.com/jmiemirza/conme/blob/HEAD/prompt_negatives/classify_llava_mix.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"88e6f2dd71a42f60"}},{"code_sha256_prefix":"7e5738752797d895","entry":"classify_contradictions","repo":"jmiemirza/conme","repo_kind":"official","path":"prompt_negatives/classify_contradictions.py","file_url":"https://github.com/jmiemirza/conme/blob/HEAD/prompt_negatives/classify_contradictions.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7e5738752797d895"}},{"code_sha256_prefix":"45c500079a8200bb","entry":"extract_items","repo":"jmiemirza/conme","repo_kind":"official","path":"utils.py","file_url":"https://github.com/jmiemirza/conme/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"45c500079a8200bb"}},{"code_sha256_prefix":"e00bb6c0ebbcf598","entry":"extract_model_performance","repo":"jmiemirza/conme","repo_kind":"official","path":"utils.py","file_url":"https://github.com/jmiemirza/conme/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e00bb6c0ebbcf598"}},{"code_sha256_prefix":"d3dac8e30e2d8874","entry":"genCommands","repo":"jmiemirza/conme","repo_kind":"official","path":"bash_scripts/eval_checklist.py","file_url":"https://github.com/jmiemirza/conme/blob/HEAD/bash_scripts/eval_checklist.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d3dac8e30e2d8874"}},{"code_sha256_prefix":"2f6456615399d387","entry":"genCommands","repo":"jmiemirza/conme","repo_kind":"official","path":"bash_scripts/train_blip.py","file_url":"https://github.com/jmiemirza/conme/blob/HEAD/bash_scripts/train_blip.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2f6456615399d387"}},{"code_sha256_prefix":"21d9bf5fbaead44b","entry":"get_model_device","repo":"jmiemirza/conme","repo_kind":"official","path":"args.py","file_url":"https://github.com/jmiemirza/conme/blob/HEAD/args.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"21d9bf5fbaead44b"}},{"code_sha256_prefix":"c4eb4d12332a6235","entry":"load_pickle","repo":"jmiemirza/conme","repo_kind":"official","path":"args.py","file_url":"https://github.com/jmiemirza/conme/blob/HEAD/args.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c4eb4d12332a6235"}},{"code_sha256_prefix":"25d79e71a7f90f8e","entry":"prompt_negative","repo":"jmiemirza/conme","repo_kind":"official","path":"prompt_negatives.py","file_url":"https://github.com/jmiemirza/conme/blob/HEAD/prompt_negatives.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"25d79e71a7f90f8e"}},{"code_sha256_prefix":"f6171fdf771d0e0c","entry":"get_args","repo":"jmiemirza/conme","repo_kind":"official","path":"args.py","file_url":"https://github.com/jmiemirza/conme/blob/HEAD/args.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f6171fdf771d0e0c"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}