{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/exposing-and-addressing-cross-task","title":"Exposing and Addressing Cross-Task Inconsistency in Unified Vision-Language Models","arxiv_id":"2303.16133","date":"2023-03-28","proceeding":null,"authors":["Adyasha Maharana","Amita Kamath","Christopher Clark","Mohit Bansal","Aniruddha Kembhavi"],"abstract":"As general purpose vision models get increasingly effective at a wide set of tasks, it is imperative that they be consistent across the tasks they support. Inconsistent AI models are considered brittle and untrustworthy by human users and are more challenging to incorporate into larger systems that take dependencies on their outputs. Measuring consistency between very heterogeneous tasks that might include outputs in different modalities is challenging since it is difficult to determine if the predictions are consistent with one another. As a solution, we introduce a benchmark dataset, CocoCon, where we create contrast sets by modifying test instances for multiple tasks in small but semantically meaningful ways to change the gold label and outline metrics for measuring if a model is consistent by ranking the original and perturbed instances across tasks. We find that state-of-the-art vision-language models suffer from a surprisingly high degree of inconsistent behavior across tasks, especially for more heterogeneous tasks. To alleviate this issue, we propose a rank correlation-based auxiliary training objective, computed over large automatically created cross-task contrast sets, that improves the multi-task consistency of large unified models while retaining their original accuracy on downstream tasks.","url_abs":"https://arxiv.org/abs/2303.16133v2","url_pdf":"https://arxiv.org/pdf/2303.16133v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"exposing-and-addressing-cross-task","repo_url":"https://github.com/adymaharana/cococon","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"jax","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[],"methods":[{"method_slug":"test","method_name":"Test"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2303.16133","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.16133"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/adymaharana/cococon","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"unverified":9},"by_repo_kind":{"official":{"samples":9,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"69563a7420a49fa8","entry":"cross_entropy_with_logits","repo":"adymaharana/cococon","repo_kind":"official","path":"unified-io/uio/model.py","file_url":"https://github.com/adymaharana/cococon/blob/HEAD/unified-io/uio/model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"69563a7420a49fa8"}},{"code_sha256_prefix":"dac0288079ac279f","entry":"dot_product_attention","repo":"adymaharana/cococon","repo_kind":"official","path":"unified-io/uio/t5x_layers.py","file_url":"https://github.com/adymaharana/cococon/blob/HEAD/unified-io/uio/t5x_layers.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"dac0288079ac279f"}},{"code_sha256_prefix":"8acbb3b977150c88","entry":"evaluate_contrast_sets","repo":"adymaharana/cococon","repo_kind":"official","path":"unified-io/evaluate_contrast_all_tasks.py","file_url":"https://github.com/adymaharana/cococon/blob/HEAD/unified-io/evaluate_contrast_all_tasks.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8acbb3b977150c88"}},{"code_sha256_prefix":"032edece79949c43","entry":"evaluate_contrast_sets","repo":"adymaharana/cococon","repo_kind":"official","path":"unified-io/evaluate_task.py","file_url":"https://github.com/adymaharana/cococon/blob/HEAD/unified-io/evaluate_task.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"032edece79949c43"}},{"code_sha256_prefix":"50fa314af669c957","entry":"get_open_img_annotations","repo":"adymaharana/cococon","repo_kind":"official","path":"unified-io/evaluate_contrast_all_tasks.py","file_url":"https://github.com/adymaharana/cococon/blob/HEAD/unified-io/evaluate_contrast_all_tasks.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"50fa314af669c957"}},{"code_sha256_prefix":"08a5061ff824fcc4","entry":"rank","repo":"adymaharana/cococon","repo_kind":"official","path":"unified-io/evaluate_contrast_all_tasks.py","file_url":"https://github.com/adymaharana/cococon/blob/HEAD/unified-io/evaluate_contrast_all_tasks.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"08a5061ff824fcc4"}},{"code_sha256_prefix":"9e2a5c7d00f0cba5","entry":"read_image","repo":"adymaharana/cococon","repo_kind":"official","path":"evaluators/evaluators.py","file_url":"https://github.com/adymaharana/cococon/blob/HEAD/evaluators/evaluators.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9e2a5c7d00f0cba5"}},{"code_sha256_prefix":"c04f426b52feac4e","entry":"reverse_space_to_depth","repo":"adymaharana/cococon","repo_kind":"official","path":"unified-io/uio/t5x_layers.py","file_url":"https://github.com/adymaharana/cococon/blob/HEAD/unified-io/uio/t5x_layers.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c04f426b52feac4e"}},{"code_sha256_prefix":"ca0c4d33f9443b8d","entry":"space_to_depth","repo":"adymaharana/cococon","repo_kind":"official","path":"unified-io/uio/t5x_layers.py","file_url":"https://github.com/adymaharana/cococon/blob/HEAD/unified-io/uio/t5x_layers.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ca0c4d33f9443b8d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}