{"url":"/sota/visual-reasoning-on-winoground","task":{"name":"Visual Reasoning","url":"/task/visual-reasoning","note":null},"dataset":{"name":"Winoground","url":"/dataset/winoground"},"category":"Computer Vision","categories":["Computer Vision","Reasoning"],"category_note":null,"description":"Ability to understand  actions and reasoning  associated with any  visual images","description_from":"task","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","rank":"the archive's row order at snapshot; not re-ranked","rows_end_at":"2025-07-28","rows_withheld_as_spam":0,"metric_values":"the archive's strings, untouched"},"metrics":["Text Score","Image Score","Group Score"],"metric_direction":{"note":"inferred from the metric name only (the archive records no direction); null = not inferred, chart draws points only","by_metric":{"Text Score":"higher","Image Score":"higher","Group Score":"higher"}},"counts":{"rows":114,"rows_with_code":78,"rows_with_paper_page":111,"rows_dated":111,"rows_using_additional_data":0},"rows":[{"rank_in_archive_order":1,"model":"GPT-4o + CA","metrics":{"Group Score":"52","Image Score":"58.5","Text Score":"75.5"},"uses_additional_data":false,"paper_date":"2025-01-23","paper":"/paper/cognitive-paradigms-for-evaluating-vlms-on","paper_url":"https://arxiv.org/abs/2501.13620v5","paper_title":"A Cognitive Paradigm Approach to Probe the Perception-Reasoning Interface in VLMs","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":2,"model":"GPT-4V (CoT, pick b/w two options)","metrics":{"Group Score":"58.75","Image Score":"68.75","Text Score":"75.25"},"uses_additional_data":false,"paper_date":"2023-11-15","paper":"/paper/the-role-of-chain-of-thought-in-complex","paper_url":"https://arxiv.org/abs/2311.09193v1","paper_title":"The Role of Chain-of-Thought in Complex Vision-Language Reasoning Task","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":3,"model":"GPT-4V (pick b/w two options)","metrics":{"Group Score":"39.25","Image Score":"46.25","Text Score":"69.25"},"uses_additional_data":false,"paper_date":"2023-11-15","paper":"/paper/the-role-of-chain-of-thought-in-complex","paper_url":"https://arxiv.org/abs/2311.09193v1","paper_title":"The Role of Chain-of-Thought in Complex Vision-Language Reasoning Task","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":4,"model":"MMICL + CoCoT","metrics":{"Group Score":"50.75","Image Score":"52.5","Text Score":"64.25"},"uses_additional_data":false,"paper_date":"2024-01-05","paper":"/paper/cocot-contrastive-chain-of-thought-prompting","paper_url":"https://arxiv.org/abs/2401.02582v1","paper_title":"CoCoT: Contrastive Chain-of-Thought Prompting for Large Multimodal Models with Multiple Image Inputs","code":"https://github.com/vista-h/gpt-4v_social_media","n_code_links":1,"syntology":null},{"rank_in_archive_order":5,"model":"GPT-4V + CoCoT","metrics":{"Group Score":"44.5","Image Score":"49.5","Text Score":"58.5"},"uses_additional_data":false,"paper_date":"2024-01-05","paper":"/paper/cocot-contrastive-chain-of-thought-prompting","paper_url":"https://arxiv.org/abs/2401.02582v1","paper_title":"CoCoT: Contrastive Chain-of-Thought Prompting for Large Multimodal Models with Multiple Image Inputs","code":"https://github.com/vista-h/gpt-4v_social_media","n_code_links":1,"syntology":null},{"rank_in_archive_order":6,"model":"OpenFlamingo + CoCoT","metrics":{"Group Score":"41.5","Image Score":"55.25","Text Score":"58.25"},"uses_additional_data":false,"paper_date":"2024-01-05","paper":"/paper/cocot-contrastive-chain-of-thought-prompting","paper_url":"https://arxiv.org/abs/2401.02582v1","paper_title":"CoCoT: Contrastive Chain-of-Thought Prompting for Large Multimodal Models with Multiple Image Inputs","code":"https://github.com/vista-h/gpt-4v_social_media","n_code_links":1,"syntology":null},{"rank_in_archive_order":7,"model":"GPT-4V","metrics":{"Group Score":"37.75","Image Score":"42.5","Text Score":"54.5"},"uses_additional_data":false,"paper_date":"2024-01-05","paper":"/paper/cocot-contrastive-chain-of-thought-prompting","paper_url":"https://arxiv.org/abs/2401.02582v1","paper_title":"CoCoT: Contrastive Chain-of-Thought Prompting for Large Multimodal Models with Multiple Image Inputs","code":"https://github.com/vista-h/gpt-4v_social_media","n_code_links":1,"syntology":null},{"rank_in_archive_order":8,"model":"FIBER (EqSim)","metrics":{"Group Score":"27.5","Image Score":"32.00","Text Score":"51.5"},"uses_additional_data":false,"paper_date":"2023-03-25","paper":"/paper/equivariant-similarity-for-vision-language","paper_url":"https://arxiv.org/abs/2303.14465v2","paper_title":"Equivariant Similarity for Vision-Language Foundation Models","code":"https://github.com/wangt-cn/eqben","n_code_links":1,"syntology":{"n_ran":1,"n_unverified":0,"n_samples":1,"n_pointer_only_licence":0}},{"rank_in_archive_order":9,"model":"FIBER (finetuned, Flickr30k)","metrics":{"Group Score":"23.00","Image Score":"26.50","Text Score":"51.25"},"uses_additional_data":false,"paper_date":"2023-03-25","paper":"/paper/equivariant-similarity-for-vision-language","paper_url":"https://arxiv.org/abs/2303.14465v2","paper_title":"Equivariant Similarity for Vision-Language Foundation Models","code":"https://github.com/wangt-cn/eqben","n_code_links":1,"syntology":{"n_ran":1,"n_unverified":0,"n_samples":1,"n_pointer_only_licence":0}},{"rank_in_archive_order":10,"model":"MMICL + CCoT","metrics":{"Group Score":"47.5","Image Score":"48","Text Score":"51"},"uses_additional_data":false,"paper_date":"2024-01-05","paper":"/paper/cocot-contrastive-chain-of-thought-prompting","paper_url":"https://arxiv.org/abs/2401.02582v1","paper_title":"CoCoT: Contrastive Chain-of-Thought Prompting for Large Multimodal Models with Multiple Image Inputs","code":"https://github.com/vista-h/gpt-4v_social_media","n_code_links":1,"syntology":null},{"rank_in_archive_order":11,"model":"OpenFlamingo + DDCoT","metrics":{"Group Score":"39","Image Score":"47.25","Text Score":"47.5"},"uses_additional_data":false,"paper_date":"2024-01-05","paper":"/paper/cocot-contrastive-chain-of-thought-prompting","paper_url":"https://arxiv.org/abs/2401.02582v1","paper_title":"CoCoT: Contrastive Chain-of-Thought Prompting for Large Multimodal Models with Multiple Image Inputs","code":"https://github.com/vista-h/gpt-4v_social_media","n_code_links":1,"syntology":null},{"rank_in_archive_order":12,"model":"VQ2","metrics":{"Group Score":"30.5","Image Score":"42.2","Text Score":"47"},"uses_additional_data":false,"paper_date":"2023-05-17","paper":"/paper/what-you-see-is-what-you-read-improving-text-1","paper_url":"https://arxiv.org/abs/2305.10400v4","paper_title":"What You See is What You Read? Improving Text-Image Alignment Evaluation","code":"https://github.com/yonatanbitton/wysiwyr","n_code_links":1,"syntology":null},{"rank_in_archive_order":13,"model":"MMICL + DDCoT","metrics":{"Group Score":"36.75","Image Score":"45","Text Score":"46.75"},"uses_additional_data":false,"paper_date":"2024-01-05","paper":"/paper/cocot-contrastive-chain-of-thought-prompting","paper_url":"https://arxiv.org/abs/2401.02582v1","paper_title":"CoCoT: Contrastive Chain-of-Thought Prompting for Large Multimodal Models with Multiple Image Inputs","code":"https://github.com/vista-h/gpt-4v_social_media","n_code_links":1,"syntology":null},{"rank_in_archive_order":14,"model":"X-VLM 16M","metrics":{"Group Score":"21.2","Image Score":"24.5","Text Score":"46.7"},"uses_additional_data":false,"paper_date":"2023-05-12","paper":"/paper/measuring-progress-in-fine-grained-vision-and","paper_url":"https://arxiv.org/abs/2305.07558v1","paper_title":"Measuring Progress in Fine-grained Vision-and-Language Understanding","code":"https://github.com/e-bug/fine-grained-evals","n_code_links":2,"syntology":{"n_ran":2,"n_unverified":2,"n_samples":4,"n_pointer_only_licence":1}},{"rank_in_archive_order":15,"model":"PaLI (ft SNLI-VE + Synthetic Data)","metrics":{"Group Score":"28.75","Image Score":"38","Text Score":"46.5"},"uses_additional_data":false,"paper_date":"2023-05-17","paper":"/paper/what-you-see-is-what-you-read-improving-text-1","paper_url":"https://arxiv.org/abs/2305.10400v4","paper_title":"What You See is What You Read? Improving Text-Image Alignment Evaluation","code":"https://github.com/yonatanbitton/wysiwyr","n_code_links":1,"syntology":null},{"rank_in_archive_order":16,"model":"FIBER","metrics":{"Group Score":"22.25","Image Score":"25.75","Text Score":"46.25"},"uses_additional_data":false,"paper_date":"2023-03-25","paper":"/paper/equivariant-similarity-for-vision-language","paper_url":"https://arxiv.org/abs/2303.14465v2","paper_title":"Equivariant Similarity for Vision-Language Foundation Models","code":"https://github.com/wangt-cn/eqben","n_code_links":1,"syntology":{"n_ran":1,"n_unverified":0,"n_samples":1,"n_pointer_only_licence":0}},{"rank_in_archive_order":17,"model":"MMICL (FLAN-T5-XXL)","metrics":{"Group Score":"43.00","Image Score":"44.99","Text Score":"45.50"},"uses_additional_data":false,"paper_date":"2023-09-14","paper":"/paper/mmicl-empowering-vision-language-model-with","paper_url":"https://arxiv.org/abs/2309.07915v3","paper_title":"MMICL: Empowering Vision-language Model with Multi-Modal In-Context Learning","code":"https://github.com/haozhezhao/mic","n_code_links":2,"syntology":{"n_ran":5,"n_unverified":2,"n_samples":7,"n_pointer_only_licence":7}},{"rank_in_archive_order":18,"model":"PaLI (ft SNLI-VE)","metrics":{"Group Score":"28.70","Image Score":"41.50","Text Score":"45.00"},"uses_additional_data":false,"paper_date":"2023-05-17","paper":"/paper/what-you-see-is-what-you-read-improving-text-1","paper_url":"https://arxiv.org/abs/2305.10400v4","paper_title":"What You See is What You Read? Improving Text-Image Alignment Evaluation","code":"https://github.com/yonatanbitton/wysiwyr","n_code_links":1,"syntology":null},{"rank_in_archive_order":19,"model":"Gemini + DDCoT","metrics":{"Group Score":"23.75","Image Score":"25","Text Score":"45"},"uses_additional_data":false,"paper_date":"2024-01-05","paper":"/paper/cocot-contrastive-chain-of-thought-prompting","paper_url":"https://arxiv.org/abs/2401.02582v1","paper_title":"CoCoT: Contrastive Chain-of-Thought Prompting for Large Multimodal Models with Multiple Image Inputs","code":"https://github.com/vista-h/gpt-4v_social_media","n_code_links":1,"syntology":null},{"rank_in_archive_order":20,"model":"METER (EqSim)","metrics":{"Group Score":"18.75","Image Score":"22.75","Text Score":"45.0"},"uses_additional_data":false,"paper_date":"2023-03-25","paper":"/paper/equivariant-similarity-for-vision-language","paper_url":"https://arxiv.org/abs/2303.14465v2","paper_title":"Equivariant Similarity for Vision-Language Foundation Models","code":"https://github.com/wangt-cn/eqben","n_code_links":1,"syntology":{"n_ran":1,"n_unverified":0,"n_samples":1,"n_pointer_only_licence":0}},{"rank_in_archive_order":21,"model":"X-VLM 4M","metrics":{"Group Score":"21.5","Image Score":"26.7","Text Score":"44.0"},"uses_additional_data":false,"paper_date":"2023-05-12","paper":"/paper/measuring-progress-in-fine-grained-vision-and","paper_url":"https://arxiv.org/abs/2305.07558v1","paper_title":"Measuring Progress in Fine-grained Vision-and-Language Understanding","code":"https://github.com/e-bug/fine-grained-evals","n_code_links":2,"syntology":{"n_ran":2,"n_unverified":2,"n_samples":4,"n_pointer_only_licence":1}},{"rank_in_archive_order":22,"model":"BLIP2 (ft COCO)","metrics":{"Group Score":"23.50","Image Score":"26.00","Text Score":"44.00"},"uses_additional_data":false,"paper_date":"2023-05-17","paper":"/paper/what-you-see-is-what-you-read-improving-text-1","paper_url":"https://arxiv.org/abs/2305.10400v4","paper_title":"What You See is What You Read? Improving Text-Image Alignment Evaluation","code":"https://github.com/yonatanbitton/wysiwyr","n_code_links":1,"syntology":null},{"rank_in_archive_order":23,"model":"KeyComp* (GPT-4)","metrics":{"Group Score":"18.2","Image Score":"28.7","Text Score":"43.5"},"uses_additional_data":false,"paper_date":"2024-01-20","paper":"/paper/prompting-large-vision-language-models-for","paper_url":"https://arxiv.org/abs/2401.11337v1","paper_title":"Prompting Large Vision-Language Models for Compositional Reasoning","code":"https://github.com/tossowski/keycomp","n_code_links":1,"syntology":{"n_ran":1,"n_unverified":1,"n_samples":2,"n_pointer_only_licence":2}},{"rank_in_archive_order":24,"model":"METER (finetuned, Flickr30k)","metrics":{"Group Score":"14.75","Image Score":"20.75","Text Score":"43.5"},"uses_additional_data":false,"paper_date":"2023-03-25","paper":"/paper/equivariant-similarity-for-vision-language","paper_url":"https://arxiv.org/abs/2303.14465v2","paper_title":"Equivariant Similarity for Vision-Language Foundation Models","code":"https://github.com/wangt-cn/eqben","n_code_links":1,"syntology":{"n_ran":1,"n_unverified":0,"n_samples":1,"n_pointer_only_licence":0}},{"rank_in_archive_order":25,"model":"BLIP2 (SGVL)","metrics":{"Group Score":"23.3","Image Score":"28.5","Text Score":"42.8"},"uses_additional_data":false,"paper_date":"2023-05-10","paper":"/paper/incorporating-structured-representations-into","paper_url":"https://arxiv.org/abs/2305.06343v2","paper_title":"Incorporating Structured Representations into Pretrained Vision & Language Models Using Scene Graphs","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":26,"model":"BLIP (SGVL)","metrics":{"Group Score":"21.5","Image Score":"27.3","Text Score":"42.8"},"uses_additional_data":false,"paper_date":"2023-05-10","paper":"/paper/incorporating-structured-representations-into","paper_url":"https://arxiv.org/abs/2305.06343v2","paper_title":"Incorporating Structured Representations into Pretrained Vision & Language Models Using Scene Graphs","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":27,"model":"KeyComp* (GPT-3.5)","metrics":{"Group Score":"17.4","Image Score":"27.8","Text Score":"42.7"},"uses_additional_data":false,"paper_date":"2024-01-20","paper":"/paper/prompting-large-vision-language-models-for","paper_url":"https://arxiv.org/abs/2401.11337v1","paper_title":"Prompting Large Vision-Language Models for Compositional Reasoning","code":"https://github.com/tossowski/keycomp","n_code_links":1,"syntology":{"n_ran":1,"n_unverified":1,"n_samples":2,"n_pointer_only_licence":2}},{"rank_in_archive_order":28,"model":"OpenFlamingo + CCoT","metrics":{"Group Score":"20","Image Score":"27.5","Text Score":"42.5"},"uses_additional_data":false,"paper_date":"2024-01-05","paper":"/paper/cocot-contrastive-chain-of-thought-prompting","paper_url":"https://arxiv.org/abs/2401.02582v1","paper_title":"CoCoT: Contrastive Chain-of-Thought Prompting for Large Multimodal Models with Multiple Image Inputs","code":"https://github.com/vista-h/gpt-4v_social_media","n_code_links":1,"syntology":null},{"rank_in_archive_order":29,"model":"NegBLIP","metrics":{"Group Score":"18.5","Image Score":"24.0","Text Score":"42.5"},"uses_additional_data":false,"paper_date":"2023-05-10","paper":"/paper/incorporating-structured-representations-into","paper_url":"https://arxiv.org/abs/2305.06343v2","paper_title":"Incorporating Structured Representations into Pretrained Vision & Language Models Using Scene Graphs","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":30,"model":"IAIS large (Flickr30k)","metrics":{"Group Score":"16.00","Image Score":"19.75","Text Score":"42.50"},"uses_additional_data":false,"paper_date":"2022-12-03","paper":"/paper/does-structural-attention-improve","paper_url":"https://sslneurips22.github.io/paper_pdfs/paper_65.pdf","paper_title":"Does Structural Attention Improve Compositional Representations in Vision-Language Models?","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":31,"model":"LLaVA-1.5-CCoT","metrics":{"Group Score":"22.3","Image Score":"35.5","Text Score":"42.0"},"uses_additional_data":false,"paper_date":"2023-11-27","paper":"/paper/compositional-chain-of-thought-prompting-for","paper_url":"https://arxiv.org/abs/2311.17076v3","paper_title":"Compositional Chain-of-Thought Prompting for Large Multimodal Models","code":"https://github.com/chancharikmitra/ccot","n_code_links":1,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":32,"model":"BLIP2","metrics":{"Group Score":"19.0","Image Score":"23.8","Text Score":"42.0"},"uses_additional_data":false,"paper_date":"2023-05-10","paper":"/paper/incorporating-structured-representations-into","paper_url":"https://arxiv.org/abs/2305.06343v2","paper_title":"Incorporating Structured Representations into Pretrained Vision & Language Models Using Scene Graphs","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":33,"model":"IAIS large (COCO)","metrics":{"Group Score":"15.50","Image Score":"19.75","Text Score":"41.75"},"uses_additional_data":false,"paper_date":"2022-12-03","paper":"/paper/does-structural-attention-improve","paper_url":"https://sslneurips22.github.io/paper_pdfs/paper_65.pdf","paper_title":"Does Structural Attention Improve Compositional Representations in Vision-Language Models?","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":34,"model":"NegBLIP2","metrics":{"Group Score":"20.5","Image Score":"26.0","Text Score":"41.5"},"uses_additional_data":false,"paper_date":"2023-05-10","paper":"/paper/incorporating-structured-representations-into","paper_url":"https://arxiv.org/abs/2305.06343v2","paper_title":"Incorporating Structured Representations into Pretrained Vision & Language Models Using Scene Graphs","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":35,"model":"BLIP (+Graph Text, +Graph Neg)","metrics":{"Group Score":"19.0","Image Score":"25.5","Text Score":"40.5"},"uses_additional_data":false,"paper_date":"2023-05-10","paper":"/paper/incorporating-structured-representations-into","paper_url":"https://arxiv.org/abs/2305.06343v2","paper_title":"Incorporating Structured Representations into Pretrained Vision & Language Models Using Scene Graphs","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":36,"model":"BLIP (+Graph Text)","metrics":{"Group Score":"16.5","Image Score":"20.5","Text Score":"40.3"},"uses_additional_data":false,"paper_date":"2023-05-10","paper":"/paper/incorporating-structured-representations-into","paper_url":"https://arxiv.org/abs/2305.06343v2","paper_title":"Incorporating Structured Representations into Pretrained Vision & Language Models Using Scene Graphs","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":37,"model":"Gemini + CoCoT","metrics":{"Group Score":"27.75","Image Score":"32.5","Text Score":"40"},"uses_additional_data":false,"paper_date":"2024-01-05","paper":"/paper/cocot-contrastive-chain-of-thought-prompting","paper_url":"https://arxiv.org/abs/2401.02582v1","paper_title":"CoCoT: Contrastive Chain-of-Thought Prompting for Large Multimodal Models with Multiple Image Inputs","code":"https://github.com/vista-h/gpt-4v_social_media","n_code_links":1,"syntology":null},{"rank_in_archive_order":38,"model":"CACR base","metrics":{"Group Score":"14.25","Image Score":"17.75","Text Score":"39.25"},"uses_additional_data":false,"paper_date":"2022-12-03","paper":"/paper/does-structural-attention-improve","paper_url":"https://sslneurips22.github.io/paper_pdfs/paper_65.pdf","paper_title":"Does Structural Attention Improve Compositional Representations in Vision-Language Models?","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":39,"model":"METER","metrics":{"Group Score":"12.00","Image Score":"15.75","Text Score":"39.25"},"uses_additional_data":false,"paper_date":"2023-03-25","paper":"/paper/equivariant-similarity-for-vision-language","paper_url":"https://arxiv.org/abs/2303.14465v2","paper_title":"Equivariant Similarity for Vision-Language Foundation Models","code":"https://github.com/wangt-cn/eqben","n_code_links":1,"syntology":{"n_ran":1,"n_unverified":0,"n_samples":1,"n_pointer_only_licence":0}},{"rank_in_archive_order":40,"model":"OpenFlamingo","metrics":{"Group Score":"33.25","Image Score":"41.25","Text Score":"39"},"uses_additional_data":false,"paper_date":"2024-01-05","paper":"/paper/cocot-contrastive-chain-of-thought-prompting","paper_url":"https://arxiv.org/abs/2401.02582v1","paper_title":"CoCoT: Contrastive Chain-of-Thought Prompting for Large Multimodal Models with Multiple Image Inputs","code":"https://github.com/vista-h/gpt-4v_social_media","n_code_links":1,"syntology":null},{"rank_in_archive_order":41,"model":"BLIP","metrics":{"Group Score":"15.0","Image Score":"19.2","Text Score":"39.0"},"uses_additional_data":false,"paper_date":"2023-05-10","paper":"/paper/incorporating-structured-representations-into","paper_url":"https://arxiv.org/abs/2305.06343v2","paper_title":"Incorporating Structured Representations into Pretrained Vision & Language Models Using Scene Graphs","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":42,"model":"GPT-4V (image-caption match answer yes/no, zero-shot)","metrics":{"Group Score":"38.00","Image Score":"38.00","Text Score":"38.00"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":43,"model":"UNITER large","metrics":{"Group Score":"10.50","Image Score":"14.00","Text Score":"38.00"},"uses_additional_data":false,"paper_date":"2022-04-07","paper":"/paper/winoground-probing-vision-and-language-models","paper_url":"https://arxiv.org/abs/2204.03162v2","paper_title":"Winoground: Probing Vision and Language Models for Visio-Linguistic Compositionality","code":"https://github.com/wangt-cn/eqben","n_code_links":2,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":44,"model":"VinVL","metrics":{"Group Score":"14.50","Image Score":"17.75","Text Score":"37.75"},"uses_additional_data":false,"paper_date":"2022-04-07","paper":"/paper/winoground-probing-vision-and-language-models","paper_url":"https://arxiv.org/abs/2204.03162v2","paper_title":"Winoground: Probing Vision and Language Models for Visio-Linguistic Compositionality","code":"https://github.com/wangt-cn/eqben","n_code_links":2,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":45,"model":"ViLLA large","metrics":{"Group Score":"11.00","Image Score":"13.25","Text Score":"37.00"},"uses_additional_data":false,"paper_date":"2022-04-07","paper":"/paper/winoground-probing-vision-and-language-models","paper_url":"https://arxiv.org/abs/2204.03162v2","paper_title":"Winoground: Probing Vision and Language Models for Visio-Linguistic Compositionality","code":"https://github.com/wangt-cn/eqben","n_code_links":2,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":46,"model":"BLIP (VisualGPTScore, α-tuned)","metrics":{"Group Score":"16.8","Image Score":"21.5","Text Score":"36.5"},"uses_additional_data":false,"paper_date":"2023-06-02","paper":"/paper/visualgptscore-visio-linguistic-reasoning","paper_url":"https://arxiv.org/abs/2306.01879v4","paper_title":"Revisiting the Role of Language Priors in Vision-Language Models","code":"https://github.com/linzhiqiu/visual_gpt_score","n_code_links":1,"syntology":null},{"rank_in_archive_order":47,"model":"BLIP 14M","metrics":{"Group Score":"14.5","Image Score":"18.5","Text Score":"36.5"},"uses_additional_data":false,"paper_date":"2023-05-12","paper":"/paper/measuring-progress-in-fine-grained-vision-and","paper_url":"https://arxiv.org/abs/2305.07558v1","paper_title":"Measuring Progress in Fine-grained Vision-and-Language Understanding","code":"https://github.com/e-bug/fine-grained-evals","n_code_links":2,"syntology":{"n_ran":2,"n_unverified":2,"n_samples":4,"n_pointer_only_licence":1}},{"rank_in_archive_order":48,"model":"ViT-B/16 + BERT base + ViLEM","metrics":{"Text Score":"36.5"},"uses_additional_data":false,"paper_date":"2023-01-01","paper":"/paper/vilem-visual-language-error-modeling-for","paper_url":"http://openaccess.thecvf.com//content/CVPR2023/html/Chen_ViLEM_Visual-Language_Error_Modeling_for_Image-Text_Retrieval_CVPR_2023_paper.html","paper_title":"ViLEM: Visual-Language Error Modeling for Image-Text Retrieval","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":49,"model":"LLaVA-1.5","metrics":{"Group Score":"20.1","Image Score":"33.3","Text Score":"36.0"},"uses_additional_data":false,"paper_date":"2023-11-27","paper":"/paper/compositional-chain-of-thought-prompting-for","paper_url":"https://arxiv.org/abs/2311.17076v3","paper_title":"Compositional Chain-of-Thought Prompting for Large Multimodal Models","code":"https://github.com/chancharikmitra/ccot","n_code_links":1,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":50,"model":"BLIP (ITM)","metrics":{"Group Score":"13.3","Image Score":"15.8","Text Score":"35.8"},"uses_additional_data":false,"paper_date":"2023-06-02","paper":"/paper/visualgptscore-visio-linguistic-reasoning","paper_url":"https://arxiv.org/abs/2306.01879v4","paper_title":"Revisiting the Role of Language Priors in Vision-Language Models","code":"https://github.com/linzhiqiu/visual_gpt_score","n_code_links":1,"syntology":null},{"rank_in_archive_order":51,"model":"BLIP 129M","metrics":{"Group Score":"11.7","Image Score":"15.0","Text Score":"35.5"},"uses_additional_data":false,"paper_date":"2023-05-12","paper":"/paper/measuring-progress-in-fine-grained-vision-and","paper_url":"https://arxiv.org/abs/2305.07558v1","paper_title":"Measuring Progress in Fine-grained Vision-and-Language Understanding","code":"https://github.com/e-bug/fine-grained-evals","n_code_links":2,"syntology":{"n_ran":2,"n_unverified":2,"n_samples":4,"n_pointer_only_licence":1}},{"rank_in_archive_order":52,"model":"ROSITA (Flickr30k)","metrics":{"Group Score":"12.25","Image Score":"15.25","Text Score":"35.25"},"uses_additional_data":false,"paper_date":"2022-12-03","paper":"/paper/does-structural-attention-improve","paper_url":"https://sslneurips22.github.io/paper_pdfs/paper_65.pdf","paper_title":"Does Structural Attention Improve Compositional Representations in Vision-Language Models?","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":53,"model":"ViLT (ViT-B/32)","metrics":{"Group Score":"9.25","Image Score":"14.00","Text Score":"34.75"},"uses_additional_data":false,"paper_date":"2022-04-07","paper":"/paper/winoground-probing-vision-and-language-models","paper_url":"https://arxiv.org/abs/2204.03162v2","paper_title":"Winoground: Probing Vision and Language Models for Visio-Linguistic Compositionality","code":"https://github.com/wangt-cn/eqben","n_code_links":2,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":54,"model":"BLIP 129M (CapFilt/L)","metrics":{"Group Score":"12.2","Image Score":"15.2","Text Score":"34.7"},"uses_additional_data":false,"paper_date":"2023-05-12","paper":"/paper/measuring-progress-in-fine-grained-vision-and","paper_url":"https://arxiv.org/abs/2305.07558v1","paper_title":"Measuring Progress in Fine-grained Vision-and-Language Understanding","code":"https://github.com/e-bug/fine-grained-evals","n_code_links":2,"syntology":{"n_ran":2,"n_unverified":2,"n_samples":4,"n_pointer_only_licence":1}},{"rank_in_archive_order":55,"model":"BLIP-ViT/L 129M","metrics":{"Group Score":"12.2","Image Score":"14.5","Text Score":"34.7"},"uses_additional_data":false,"paper_date":"2023-05-12","paper":"/paper/measuring-progress-in-fine-grained-vision-and","paper_url":"https://arxiv.org/abs/2305.07558v1","paper_title":"Measuring Progress in Fine-grained Vision-and-Language Understanding","code":"https://github.com/e-bug/fine-grained-evals","n_code_links":2,"syntology":{"n_ran":2,"n_unverified":2,"n_samples":4,"n_pointer_only_licence":1}},{"rank_in_archive_order":56,"model":"Diffusion Classifier (zero-shot)","metrics":{"Text Score":"34.00"},"uses_additional_data":false,"paper_date":"2023-03-28","paper":"/paper/your-diffusion-model-is-secretly-a-zero-shot","paper_url":"https://arxiv.org/abs/2303.16203v3","paper_title":"Your Diffusion Model is Secretly a Zero-Shot Classifier","code":"https://github.com/diffusion-classifier/diffusion-classifier","n_code_links":4,"syntology":{"n_ran":2,"n_unverified":0,"n_samples":2,"n_pointer_only_licence":1}},{"rank_in_archive_order":57,"model":"PEVL 14M","metrics":{"Group Score":"12.2","Image Score":"15.7","Text Score":"33.2"},"uses_additional_data":false,"paper_date":"2023-05-12","paper":"/paper/measuring-progress-in-fine-grained-vision-and","paper_url":"https://arxiv.org/abs/2305.07558v1","paper_title":"Measuring Progress in Fine-grained Vision-and-Language Understanding","code":"https://github.com/e-bug/fine-grained-evals","n_code_links":2,"syntology":{"n_ran":2,"n_unverified":2,"n_samples":4,"n_pointer_only_licence":1}},{"rank_in_archive_order":58,"model":"ALBEF 14M","metrics":{"Group Score":"12.7","Image Score":"16.2","Text Score":"32.5"},"uses_additional_data":false,"paper_date":"2023-05-12","paper":"/paper/measuring-progress-in-fine-grained-vision-and","paper_url":"https://arxiv.org/abs/2305.07558v1","paper_title":"Measuring Progress in Fine-grained Vision-and-Language Understanding","code":"https://github.com/e-bug/fine-grained-evals","n_code_links":2,"syntology":{"n_ran":2,"n_unverified":2,"n_samples":4,"n_pointer_only_licence":1}},{"rank_in_archive_order":59,"model":"FLAVA (ITM)","metrics":{"Group Score":"14.25","Image Score":"20.50","Text Score":"32.25"},"uses_additional_data":false,"paper_date":"2022-04-07","paper":"/paper/winoground-probing-vision-and-language-models","paper_url":"https://arxiv.org/abs/2204.03162v2","paper_title":"Winoground: Probing Vision and Language Models for Visio-Linguistic Compositionality","code":"https://github.com/wangt-cn/eqben","n_code_links":2,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":60,"model":"UNITER base","metrics":{"Group Score":"10.00","Image Score":"13.25","Text Score":"32.25"},"uses_additional_data":false,"paper_date":"2022-04-07","paper":"/paper/winoground-probing-vision-and-language-models","paper_url":"https://arxiv.org/abs/2204.03162v2","paper_title":"Winoground: Probing Vision and Language Models for Visio-Linguistic Compositionality","code":"https://github.com/wangt-cn/eqben","n_code_links":2,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":61,"model":"CLIP (SGVL)","metrics":{"Group Score":"9.8","Image Score":"14.0","Text Score":"32.0"},"uses_additional_data":false,"paper_date":"2023-05-10","paper":"/paper/incorporating-structured-representations-into","paper_url":"https://arxiv.org/abs/2305.06343v2","paper_title":"Incorporating Structured Representations into Pretrained Vision & Language Models Using Scene Graphs","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":62,"model":"ViT-B/16 + BERT base","metrics":{"Text Score":"31.2"},"uses_additional_data":false,"paper_date":"2023-01-01","paper":"/paper/vilem-visual-language-error-modeling-for","paper_url":"http://openaccess.thecvf.com//content/CVPR2023/html/Chen_ViLEM_Visual-Language_Error_Modeling_for_Image-Text_Retrieval_CVPR_2023_paper.html","paper_title":"ViLEM: Visual-Language Error Modeling for Image-Text Retrieval","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":63,"model":"Gemini","metrics":{"Group Score":"25","Image Score":"26","Text Score":"30.75"},"uses_additional_data":false,"paper_date":"2024-01-05","paper":"/paper/cocot-contrastive-chain-of-thought-prompting","paper_url":"https://arxiv.org/abs/2401.02582v1","paper_title":"CoCoT: Contrastive Chain-of-Thought Prompting for Large Multimodal Models with Multiple Image Inputs","code":"https://github.com/vista-h/gpt-4v_social_media","n_code_links":1,"syntology":null},{"rank_in_archive_order":64,"model":"OCLIP (ViT-H/14)","metrics":{"Image Score":"12.75","Text Score":"30.75"},"uses_additional_data":false,"paper_date":"2023-11-17","paper":"/paper/selfeval-leveraging-the-discriminative-nature","paper_url":"https://arxiv.org/abs/2311.10708v2","paper_title":"SelfEval: Leveraging the discriminative nature of generative models for evaluation","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":65,"model":"CLIP (ViT-B/32)","metrics":{"Group Score":"8.00","Image Score":"10.50","Text Score":"30.75"},"uses_additional_data":false,"paper_date":"2022-04-07","paper":"/paper/winoground-probing-vision-and-language-models","paper_url":"https://arxiv.org/abs/2204.03162v2","paper_title":"Winoground: Probing Vision and Language Models for Visio-Linguistic Compositionality","code":"https://github.com/wangt-cn/eqben","n_code_links":2,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":66,"model":"OFA large (ITM)","metrics":{"Group Score":"7.25","Image Score":"10.25","Text Score":"30.75"},"uses_additional_data":false,"paper_date":"2023-05-11","paper":"/paper/simple-token-level-confidence-improves","paper_url":"https://arxiv.org/abs/2305.07021v1","paper_title":"Simple Token-Level Confidence Improves Caption Correctness","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":67,"model":"KeyComp (GPT-3.5)","metrics":{"Group Score":"12.4","Image Score":"24.6","Text Score":"30.3"},"uses_additional_data":false,"paper_date":"2024-01-20","paper":"/paper/prompting-large-vision-language-models-for","paper_url":"https://arxiv.org/abs/2401.11337v1","paper_title":"Prompting Large Vision-Language Models for Compositional Reasoning","code":"https://github.com/tossowski/keycomp","n_code_links":1,"syntology":{"n_ran":1,"n_unverified":1,"n_samples":2,"n_pointer_only_licence":2}},{"rank_in_archive_order":68,"model":"CLIP (ViT-L/14)","metrics":{"Image Score":"8.0","Text Score":"30.25"},"uses_additional_data":false,"paper_date":"2023-11-17","paper":"/paper/selfeval-leveraging-the-discriminative-nature","paper_url":"https://arxiv.org/abs/2311.10708v2","paper_title":"SelfEval: Leveraging the discriminative nature of generative models for evaluation","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":69,"model":"ViLLA base","metrics":{"Group Score":"8.00","Image Score":"12.00","Text Score":"30.00"},"uses_additional_data":false,"paper_date":"2022-04-07","paper":"/paper/winoground-probing-vision-and-language-models","paper_url":"https://arxiv.org/abs/2204.03162v2","paper_title":"Winoground: Probing Vision and Language Models for Visio-Linguistic Compositionality","code":"https://github.com/wangt-cn/eqben","n_code_links":2,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":70,"model":"syn-CLIP","metrics":{"Group Score":"9.50","Image Score":"11.50","Text Score":"30.00"},"uses_additional_data":false,"paper_date":"2023-03-30","paper":"/paper/going-beyond-nouns-with-vision-language","paper_url":"https://arxiv.org/abs/2303.17590v2","paper_title":"Going Beyond Nouns With Vision & Language Models Using Synthetic Data","code":"https://github.com/uvavision/syvic","n_code_links":1,"syntology":null},{"rank_in_archive_order":71,"model":"syn-CyCLIP","metrics":{"Group Score":"8.25","Image Score":"10.75","Text Score":"30.00"},"uses_additional_data":false,"paper_date":"2023-03-30","paper":"/paper/going-beyond-nouns-with-vision-language","paper_url":"https://arxiv.org/abs/2303.17590v2","paper_title":"Going Beyond Nouns With Vision & Language Models Using Synthetic Data","code":"https://github.com/uvavision/syvic","n_code_links":1,"syntology":null},{"rank_in_archive_order":72,"model":"NegCLIP","metrics":{"Group Score":"8.0","Image Score":"10.5","Text Score":"29.5"},"uses_additional_data":false,"paper_date":"2023-05-10","paper":"/paper/incorporating-structured-representations-into","paper_url":"https://arxiv.org/abs/2305.06343v2","paper_title":"Incorporating Structured Representations into Pretrained Vision & Language Models Using Scene Graphs","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":73,"model":"OFA large (TLC-A)","metrics":{"Group Score":"17.50","Image Score":"27.00","Text Score":"29.25"},"uses_additional_data":false,"paper_date":"2023-05-11","paper":"/paper/simple-token-level-confidence-improves","paper_url":"https://arxiv.org/abs/2305.07021v1","paper_title":"Simple Token-Level Confidence Improves Caption Correctness","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":74,"model":"ALBEF 4M","metrics":{"Group Score":"11.0","Image Score":"15.5","Text Score":"29.2"},"uses_additional_data":false,"paper_date":"2023-05-12","paper":"/paper/measuring-progress-in-fine-grained-vision-and","paper_url":"https://arxiv.org/abs/2305.07558v1","paper_title":"Measuring Progress in Fine-grained Vision-and-Language Understanding","code":"https://github.com/e-bug/fine-grained-evals","n_code_links":2,"syntology":{"n_ran":2,"n_unverified":2,"n_samples":4,"n_pointer_only_licence":1}},{"rank_in_archive_order":75,"model":"LDM-T5 (SelfEval)","metrics":{"Image Score":"13.50","Text Score":"29.00"},"uses_additional_data":false,"paper_date":"2023-11-17","paper":"/paper/selfeval-leveraging-the-discriminative-nature","paper_url":"https://arxiv.org/abs/2311.10708v2","paper_title":"SelfEval: Leveraging the discriminative nature of generative models for evaluation","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":76,"model":"CyCLIP","metrics":{"Group Score":"7.25","Image Score":"9.50","Text Score":"28.50"},"uses_additional_data":false,"paper_date":"2023-03-30","paper":"/paper/going-beyond-nouns-with-vision-language","paper_url":"https://arxiv.org/abs/2303.17590v2","paper_title":"Going Beyond Nouns With Vision & Language Models Using Synthetic Data","code":"https://github.com/uvavision/syvic","n_code_links":1,"syntology":null},{"rank_in_archive_order":77,"model":"PDM-T5 (SelfEval)","metrics":{"Image Score":"12.00","Text Score":"28.25"},"uses_additional_data":false,"paper_date":"2023-11-17","paper":"/paper/selfeval-leveraging-the-discriminative-nature","paper_url":"https://arxiv.org/abs/2311.10708v2","paper_title":"SelfEval: Leveraging the discriminative nature of generative models for evaluation","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":78,"model":"COCA ViT-L14 (f.t on COCO)","metrics":{"Group Score":"8.25","Image Score":"11.50","Text Score":"28.25"},"uses_additional_data":false,"paper_date":"2023-05-17","paper":"/paper/what-you-see-is-what-you-read-improving-text-1","paper_url":"https://arxiv.org/abs/2305.10400v4","paper_title":"What You See is What You Read? Improving Text-Image Alignment Evaluation","code":"https://github.com/yonatanbitton/wysiwyr","n_code_links":1,"syntology":null},{"rank_in_archive_order":79,"model":"LLaVA-1.5-ZS-CoT","metrics":{"Group Score":"12.3","Image Score":"22.5","Text Score":"28.0"},"uses_additional_data":false,"paper_date":"2023-11-27","paper":"/paper/compositional-chain-of-thought-prompting-for","paper_url":"https://arxiv.org/abs/2311.17076v3","paper_title":"Compositional Chain-of-Thought Prompting for Large Multimodal Models","code":"https://github.com/chancharikmitra/ccot","n_code_links":1,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":80,"model":"BLIP (ITC)","metrics":{"Group Score":"6.5","Image Score":"9.0","Text Score":"28.0"},"uses_additional_data":false,"paper_date":"2023-06-02","paper":"/paper/visualgptscore-visio-linguistic-reasoning","paper_url":"https://arxiv.org/abs/2306.01879v4","paper_title":"Revisiting the Role of Language Priors in Vision-Language Models","code":"https://github.com/linzhiqiu/visual_gpt_score","n_code_links":1,"syntology":null},{"rank_in_archive_order":81,"model":"OFA large (ft SNLI-VE)","metrics":{"Group Score":"9.00","Image Score":"14.30","Text Score":"27.70"},"uses_additional_data":false,"paper_date":"2023-05-17","paper":"/paper/what-you-see-is-what-you-read-improving-text-1","paper_url":"https://arxiv.org/abs/2305.10400v4","paper_title":"What You See is What You Read? Improving Text-Image Alignment Evaluation","code":"https://github.com/yonatanbitton/wysiwyr","n_code_links":1,"syntology":null},{"rank_in_archive_order":82,"model":"OFA base (ITM)","metrics":{"Group Score":"6.50","Image Score":"10.75","Text Score":"26.75"},"uses_additional_data":false,"paper_date":"2023-05-11","paper":"/paper/simple-token-level-confidence-improves","paper_url":"https://arxiv.org/abs/2305.07021v1","paper_title":"Simple Token-Level Confidence Improves Caption Correctness","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":83,"model":"CLIP RN50x64","metrics":{"Group Score":"10.25","Image Score":"13.75","Text Score":"26.50"},"uses_additional_data":false,"paper_date":"2023-05-17","paper":"/paper/what-you-see-is-what-you-read-improving-text-1","paper_url":"https://arxiv.org/abs/2305.10400v4","paper_title":"What You See is What You Read? Improving Text-Image Alignment Evaluation","code":"https://github.com/yonatanbitton/wysiwyr","n_code_links":1,"syntology":null},{"rank_in_archive_order":84,"model":"LLaVA-7B (GPTScore)","metrics":{"Group Score":"10.50","Image Score":"17.00","Text Score":"25.50"},"uses_additional_data":false,"paper_date":"2023-08-21","paper":"/paper/an-examination-of-the-compositionality-of","paper_url":"https://arxiv.org/abs/2308.10509v2","paper_title":"An Examination of the Compositionality of Large Generative Vision-Language Models","code":"https://github.com/teleema/sade","n_code_links":1,"syntology":null},{"rank_in_archive_order":85,"model":"FLAVA (contrastive)","metrics":{"Group Score":"9.00","Image Score":"13.50","Text Score":"25.25"},"uses_additional_data":false,"paper_date":"2022-04-07","paper":"/paper/winoground-probing-vision-and-language-models","paper_url":"https://arxiv.org/abs/2204.03162v2","paper_title":"Winoground: Probing Vision and Language Models for Visio-Linguistic Compositionality","code":"https://github.com/wangt-cn/eqben","n_code_links":2,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":86,"model":"Random chance","metrics":{"Group Score":"16.67","Image Score":"25.00","Text Score":"25.00"},"uses_additional_data":false,"paper_date":"2022-04-07","paper":"/paper/winoground-probing-vision-and-language-models","paper_url":"https://arxiv.org/abs/2204.03162v2","paper_title":"Winoground: Probing Vision and Language Models for Visio-Linguistic Compositionality","code":"https://github.com/wangt-cn/eqben","n_code_links":2,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":87,"model":"LLaVA","metrics":{"Group Score":"13.0","Image Score":"25.0","Text Score":"24.8"},"uses_additional_data":false,"paper_date":"2023-05-10","paper":"/paper/incorporating-structured-representations-into","paper_url":"https://arxiv.org/abs/2305.06343v2","paper_title":"Incorporating Structured Representations into Pretrained Vision & Language Models Using Scene Graphs","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":88,"model":"OFA base (TLC-A)","metrics":{"Group Score":"13.75","Image Score":"23.50","Text Score":"24.50"},"uses_additional_data":false,"paper_date":"2023-05-11","paper":"/paper/simple-token-level-confidence-improves","paper_url":"https://arxiv.org/abs/2305.07021v1","paper_title":"Simple Token-Level Confidence Improves Caption Correctness","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":89,"model":"MiniGPT-4-7B (GPTScore)","metrics":{"Group Score":"11.50","Image Score":"21.75","Text Score":"24.50"},"uses_additional_data":false,"paper_date":"2023-08-21","paper":"/paper/an-examination-of-the-compositionality-of","paper_url":"https://arxiv.org/abs/2308.10509v2","paper_title":"An Examination of the Compositionality of Large Generative Vision-Language Models","code":"https://github.com/teleema/sade","n_code_links":1,"syntology":null},{"rank_in_archive_order":90,"model":"ViLBERT base","metrics":{"Group Score":"4.75","Image Score":"7.25","Text Score":"23.75"},"uses_additional_data":false,"paper_date":"2022-04-07","paper":"/paper/winoground-probing-vision-and-language-models","paper_url":"https://arxiv.org/abs/2204.03162v2","paper_title":"Winoground: Probing Vision and Language Models for Visio-Linguistic Compositionality","code":"https://github.com/wangt-cn/eqben","n_code_links":2,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":91,"model":"MiniGPT-4","metrics":{"Group Score":"9.5","Image Score":"18.0","Text Score":"23.3"},"uses_additional_data":false,"paper_date":"2023-05-10","paper":"/paper/incorporating-structured-representations-into","paper_url":"https://arxiv.org/abs/2305.06343v2","paper_title":"Incorporating Structured Representations into Pretrained Vision & Language Models Using Scene Graphs","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":92,"model":"MiniGPT-4-7B (VisualGPTScore)","metrics":{"Group Score":"9.50","Image Score":"18.00","Text Score":"23.25"},"uses_additional_data":false,"paper_date":"2023-08-21","paper":"/paper/an-examination-of-the-compositionality-of","paper_url":"https://arxiv.org/abs/2308.10509v2","paper_title":"An Examination of the Compositionality of Large Generative Vision-Language Models","code":"https://github.com/teleema/sade","n_code_links":1,"syntology":null},{"rank_in_archive_order":93,"model":"VSE++ (COCO, ResNet)","metrics":{"Group Score":"4.00","Image Score":"8.00","Text Score":"22.75"},"uses_additional_data":false,"paper_date":"2022-04-07","paper":"/paper/winoground-probing-vision-and-language-models","paper_url":"https://arxiv.org/abs/2204.03162v2","paper_title":"Winoground: Probing Vision and Language Models for Visio-Linguistic Compositionality","code":"https://github.com/wangt-cn/eqben","n_code_links":2,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":94,"model":"OFA tiny (ITM)","metrics":{"Group Score":"4.50","Image Score":"7.75","Text Score":"22.75"},"uses_additional_data":false,"paper_date":"2023-05-11","paper":"/paper/simple-token-level-confidence-improves","paper_url":"https://arxiv.org/abs/2305.07021v1","paper_title":"Simple Token-Level Confidence Improves Caption Correctness","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":95,"model":"LDM-CLIP (SelfEval)","metrics":{"Image Score":"7.25","Text Score":"22.75"},"uses_additional_data":false,"paper_date":"2023-11-17","paper":"/paper/selfeval-leveraging-the-discriminative-nature","paper_url":"https://arxiv.org/abs/2311.10708v2","paper_title":"SelfEval: Leveraging the discriminative nature of generative models for evaluation","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":96,"model":"Gemini + CCoT","metrics":{"Group Score":"20.75","Image Score":"33","Text Score":"22.5"},"uses_additional_data":false,"paper_date":"2024-01-05","paper":"/paper/cocot-contrastive-chain-of-thought-prompting","paper_url":"https://arxiv.org/abs/2401.02582v1","paper_title":"CoCoT: Contrastive Chain-of-Thought Prompting for Large Multimodal Models with Multiple Image Inputs","code":"https://github.com/vista-h/gpt-4v_social_media","n_code_links":1,"syntology":null},{"rank_in_archive_order":97,"model":"InstructBLIP-CCoT","metrics":{"Group Score":"8.3","Image Score":"21.3","Text Score":"21.0"},"uses_additional_data":false,"paper_date":"2023-11-27","paper":"/paper/compositional-chain-of-thought-prompting-for","paper_url":"https://arxiv.org/abs/2311.17076v3","paper_title":"Compositional Chain-of-Thought Prompting for Large Multimodal Models","code":"https://github.com/chancharikmitra/ccot","n_code_links":1,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":98,"model":"VSRN (Flickr30k)","metrics":{"Group Score":"3.50","Image Score":"5.00","Text Score":"20.00"},"uses_additional_data":false,"paper_date":"2022-04-07","paper":"/paper/winoground-probing-vision-and-language-models","paper_url":"https://arxiv.org/abs/2204.03162v2","paper_title":"Winoground: Probing Vision and Language Models for Visio-Linguistic Compositionality","code":"https://github.com/wangt-cn/eqben","n_code_links":2,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":99,"model":"VSE++ (Flickr30k, ResNet)","metrics":{"Group Score":"2.75","Image Score":"5.00","Text Score":"20.00"},"uses_additional_data":false,"paper_date":"2022-04-07","paper":"/paper/winoground-probing-vision-and-language-models","paper_url":"https://arxiv.org/abs/2204.03162v2","paper_title":"Winoground: Probing Vision and Language Models for Visio-Linguistic Compositionality","code":"https://github.com/wangt-cn/eqben","n_code_links":2,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":100,"model":"VSE++ (Flickr30k, VGG)","metrics":{"Group Score":"4.50","Image Score":"6.25","Text Score":"19.75"},"uses_additional_data":false,"paper_date":"2022-04-07","paper":"/paper/winoground-probing-vision-and-language-models","paper_url":"https://arxiv.org/abs/2204.03162v2","paper_title":"Winoground: Probing Vision and Language Models for Visio-Linguistic Compositionality","code":"https://github.com/wangt-cn/eqben","n_code_links":2,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":101,"model":"UniT (ITM finetuned)","metrics":{"Group Score":"4.00","Image Score":"6.25","Text Score":"19.50"},"uses_additional_data":false,"paper_date":"2022-04-07","paper":"/paper/winoground-probing-vision-and-language-models","paper_url":"https://arxiv.org/abs/2204.03162v2","paper_title":"Winoground: Probing Vision and Language Models for Visio-Linguistic Compositionality","code":"https://github.com/wangt-cn/eqben","n_code_links":2,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":102,"model":"LXMERT","metrics":{"Group Score":"4.00","Image Score":"7.00","Text Score":"19.25"},"uses_additional_data":false,"paper_date":"2022-04-07","paper":"/paper/winoground-probing-vision-and-language-models","paper_url":"https://arxiv.org/abs/2204.03162v2","paper_title":"Winoground: Probing Vision and Language Models for Visio-Linguistic Compositionality","code":"https://github.com/wangt-cn/eqben","n_code_links":2,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":103,"model":"TIFA","metrics":{"Group Score":"11.30","Image Score":"12.50","Text Score":"19.00"},"uses_additional_data":false,"paper_date":"2023-05-17","paper":"/paper/what-you-see-is-what-you-read-improving-text-1","paper_url":"https://arxiv.org/abs/2305.10400v4","paper_title":"What You See is What You Read? Improving Text-Image Alignment Evaluation","code":"https://github.com/yonatanbitton/wysiwyr","n_code_links":1,"syntology":null},{"rank_in_archive_order":104,"model":"IDEFICS 80B","metrics":{"Group Score":"8.0","Image Score":"22.5","Text Score":"18.75"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":105,"model":"VSE++ (COCO, VGG)","metrics":{"Group Score":"3.50","Image Score":"5.50","Text Score":"18.75"},"uses_additional_data":false,"paper_date":"2022-04-07","paper":"/paper/winoground-probing-vision-and-language-models","paper_url":"https://arxiv.org/abs/2204.03162v2","paper_title":"Winoground: Probing Vision and Language Models for Visio-Linguistic Compositionality","code":"https://github.com/wangt-cn/eqben","n_code_links":2,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":106,"model":"VSRN (COCO)","metrics":{"Group Score":"3.75","Image Score":"7.00","Text Score":"17.50"},"uses_additional_data":false,"paper_date":"2022-04-07","paper":"/paper/winoground-probing-vision-and-language-models","paper_url":"https://arxiv.org/abs/2204.03162v2","paper_title":"Winoground: Probing Vision and Language Models for Visio-Linguistic Compositionality","code":"https://github.com/wangt-cn/eqben","n_code_links":2,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":107,"model":"PDM-CLIP (SelfEval)","metrics":{"Image Score":"14.00","Text Score":"17.00"},"uses_additional_data":false,"paper_date":"2023-11-17","paper":"/paper/selfeval-leveraging-the-discriminative-nature","paper_url":"https://arxiv.org/abs/2311.10708v2","paper_title":"SelfEval: Leveraging the discriminative nature of generative models for evaluation","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":108,"model":"IDEFICS 9B","metrics":{"Group Score":"5.0","Image Score":"20.8","Text Score":"16.8"},"uses_additional_data":false,"paper_date":null,"paper":null,"paper_url":null,"paper_title":"","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":109,"model":"OFA tiny (TLC-A)","metrics":{"Group Score":"6.75","Image Score":"15.75","Text Score":"16.50"},"uses_additional_data":false,"paper_date":"2023-05-11","paper":"/paper/simple-token-level-confidence-improves","paper_url":"https://arxiv.org/abs/2305.07021v1","paper_title":"Simple Token-Level Confidence Improves Caption Correctness","code":null,"n_code_links":0,"syntology":null},{"rank_in_archive_order":110,"model":"VisualBERT base","metrics":{"Group Score":"1.50","Image Score":"2.50","Text Score":"15.50"},"uses_additional_data":false,"paper_date":"2022-04-07","paper":"/paper/winoground-probing-vision-and-language-models","paper_url":"https://arxiv.org/abs/2204.03162v2","paper_title":"Winoground: Probing Vision and Language Models for Visio-Linguistic Compositionality","code":"https://github.com/wangt-cn/eqben","n_code_links":2,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":111,"model":"MiniGPT-4-7B (BERTScore)","metrics":{"Group Score":"2.75","Image Score":"8.00","Text Score":"14.00"},"uses_additional_data":false,"paper_date":"2023-08-21","paper":"/paper/an-examination-of-the-compositionality-of","paper_url":"https://arxiv.org/abs/2308.10509v2","paper_title":"An Examination of the Compositionality of Large Generative Vision-Language Models","code":"https://github.com/teleema/sade","n_code_links":1,"syntology":null},{"rank_in_archive_order":112,"model":"LLaVA-7B (BERTScore)","metrics":{"Group Score":"2.25","Image Score":"5.25","Text Score":"13.50"},"uses_additional_data":false,"paper_date":"2023-08-21","paper":"/paper/an-examination-of-the-compositionality-of","paper_url":"https://arxiv.org/abs/2308.10509v2","paper_title":"An Examination of the Compositionality of Large Generative Vision-Language Models","code":"https://github.com/teleema/sade","n_code_links":1,"syntology":null},{"rank_in_archive_order":113,"model":"InstructBLIP-ZS-CoT","metrics":{"Group Score":"4.0","Image Score":"16.3","Text Score":"9.3"},"uses_additional_data":false,"paper_date":"2023-11-27","paper":"/paper/compositional-chain-of-thought-prompting-for","paper_url":"https://arxiv.org/abs/2311.17076v3","paper_title":"Compositional Chain-of-Thought Prompting for Large Multimodal Models","code":"https://github.com/chancharikmitra/ccot","n_code_links":1,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}},{"rank_in_archive_order":114,"model":"InstructBLIP","metrics":{"Group Score":"3.3","Image Score":"11.5","Text Score":"7.0"},"uses_additional_data":false,"paper_date":"2023-11-27","paper":"/paper/compositional-chain-of-thought-prompting-for","paper_url":"https://arxiv.org/abs/2311.17076v3","paper_title":"Compositional Chain-of-Thought Prompting for Large Multimodal Models","code":"https://github.com/chancharikmitra/ccot","n_code_links":1,"syntology":{"n_ran":3,"n_unverified":1,"n_samples":4,"n_pointer_only_licence":0}}],"since_archive":{"claim":"Results that newer papers report for their own method, placed here by Syntology. A model pointed at the cell in the paper's own table; the number was read from that cell and checked against this leaderboard's metric, dataset, split and scale; an independent check that saw this leaderboard's other rows and every other leaderboard on the same dataset accepted it. Not reviewed by the paper's authors or by the archive's editors, and not ranked against the archive rows.","extraction_file_present":true,"measurement":{"test_papers":883,"papers_with_output":881,"judged_true":108,"judged":110,"wilson95_lower":0.9361,"measured_on":"2026-09-24","frozen_commit":"0e3de0df94"},"measurement_note":"blind adjudication of accepted entries on a held-out split of archive papers, rules frozen before the test","coverage":{"sentence":"Syntology has checked 6,264 of the 9,581 papers on this site that are newer than the archive; results from the others appear after they are checked.","complete":false,"papers_newer_than_archive":9581,"papers_checked":6264,"papers_extracted_not_yet_verified":0,"boards_without_verdict":2,"papers_not_yet_extracted":3316},"order":"newest first by month (arXiv date, else the arXiv-id month), then arXiv id descending","columns":[],"entries":[]},"syntology":{"read_at":"2026-09-24T18:15:14+00:00","claim":"Per row: N of M harvested code samples from that row's paper executed on a synthesized fixture; the other M-N are unverified. Not a reproduction of the row's number; not a correctness claim. n_pointer_only_licence counts samples the site points at rather than redistributes (a licence axis, independent of ran/unverified).","rows_with_graph_line":46,"rows_with_any_sample_ran":46,"distinct_papers_with_graph_line":7,"distinct_papers_with_any_sample_ran":7,"samples_over_distinct_papers":{"n_ran":17,"n_unverified":7,"n_samples":24,"n_pointer_only_licence":11,"note":"each paper (arXiv id) counted once, however many rows it is behind; this is the page-level figure"},"samples_row_weighted":{"n_ran":112,"n_unverified":49,"n_samples":161,"n_pointer_only_licence":23,"note":"row-weighted: a paper behind several rows is counted once per row; inflated relative to samples_over_distinct_papers by design, kept for readers summing the per-row syntology blocks"}}}