{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/a-joint-study-of-phrase-grounding-and-task","title":"A Joint Study of Phrase Grounding and Task Performance in Vision and Language Models","arxiv_id":"2309.02691","date":"2023-09-06","proceeding":null,"authors":["Noriyuki Kojima","Hadar Averbuch-Elor","Yoav Artzi"],"abstract":"Key to tasks that require reasoning about natural language in visual contexts is grounding words and phrases to image regions. However, observing this grounding in contemporary models is complex, even if it is generally expected to take place if the task is addressed in a way that is conductive to generalization. We propose a framework to jointly study task performance and phrase grounding, and propose three benchmarks to study the relation between the two. Our results show that contemporary models demonstrate inconsistency between their ability to ground phrases and solve tasks. We show how this can be addressed through brute-force training on ground phrasing annotations, and analyze the dynamics it creates. Code and at available at https://github.com/lil-lab/phrase_grounding.","url_abs":"https://arxiv.org/abs/2309.02691v3","url_pdf":"https://arxiv.org/pdf/2309.02691v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"a-joint-study-of-phrase-grounding-and-task","repo_url":"https://github.com/lil-lab/phrase_grounding","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"phrase-grounding","task_name":"Phrase Grounding"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2309.02691","atlas_url":"https://app.syntology.ai/?focus=2309.02691","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2309.02691"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/lil-lab/phrase_grounding","reach":{"status":"ok"}}],"summary":{"ran":9,"ran_honours":1,"unverified":2},"by_repo_kind":{"official":{"samples":12,"ran":10,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":12,"samples":[{"code_sha256_prefix":"49dd5b60a076a608","entry":"box_abs_to_rel","repo":"lil-lab/phrase_grounding","repo_kind":"official","path":"src/datasets/coco_format_dataset.py","file_url":"https://github.com/lil-lab/phrase_grounding/blob/HEAD/src/datasets/coco_format_dataset.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"49dd5b60a076a608"}},{"code_sha256_prefix":"03b8664c2bd40c24","entry":"box_area","repo":"lil-lab/phrase_grounding","repo_kind":"official","path":"src/evals/utils/grounding_eval.py","file_url":"https://github.com/lil-lab/phrase_grounding/blob/HEAD/src/evals/utils/grounding_eval.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"03b8664c2bd40c24"}},{"code_sha256_prefix":"e0a06ded5d4f6c3c","entry":"box_cxcywh_to_xyxy","repo":"lil-lab/phrase_grounding","repo_kind":"official","path":"src/evals/postprocessors.py","file_url":"https://github.com/lil-lab/phrase_grounding/blob/HEAD/src/evals/postprocessors.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e0a06ded5d4f6c3c"}},{"code_sha256_prefix":"ac92d3ba92f8d1ef","entry":"box_cxcywh_to_xyxy","repo":"lil-lab/phrase_grounding","repo_kind":"official","path":"src/datasets/coco_format_dataset.py","file_url":"https://github.com/lil-lab/phrase_grounding/blob/HEAD/src/datasets/coco_format_dataset.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ac92d3ba92f8d1ef"}},{"code_sha256_prefix":"da25def25e73d0bc","entry":"box_rel_to_abs","repo":"lil-lab/phrase_grounding","repo_kind":"official","path":"src/datasets/coco_format_dataset.py","file_url":"https://github.com/lil-lab/phrase_grounding/blob/HEAD/src/datasets/coco_format_dataset.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"da25def25e73d0bc"}},{"code_sha256_prefix":"2a4c77b56ecc93cc","entry":"convert_to_dict_format","repo":"lil-lab/phrase_grounding","repo_kind":"official","path":"src/evals/db_utils.py","file_url":"https://github.com/lil-lab/phrase_grounding/blob/HEAD/src/evals/db_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2a4c77b56ecc93cc"}},{"code_sha256_prefix":"5edea330b9255674","entry":"convert_to_dict_format","repo":"lil-lab/phrase_grounding","repo_kind":"official","path":"src/evals/evaluators.py","file_url":"https://github.com/lil-lab/phrase_grounding/blob/HEAD/src/evals/evaluators.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5edea330b9255674"}},{"code_sha256_prefix":"019b3a45a6d11ffb","entry":"discretize_boxes","repo":"lil-lab/phrase_grounding","repo_kind":"official","path":"src/evals/utils/grounding_eval.py","file_url":"https://github.com/lil-lab/phrase_grounding/blob/HEAD/src/evals/utils/grounding_eval.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"019b3a45a6d11ffb"}},{"code_sha256_prefix":"570653f683a7d0ae","entry":"round_bboxes","repo":"lil-lab/phrase_grounding","repo_kind":"official","path":"src/evals/utils/grounding_eval.py","file_url":"https://github.com/lil-lab/phrase_grounding/blob/HEAD/src/evals/utils/grounding_eval.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"570653f683a7d0ae"}},{"code_sha256_prefix":"df66d21a558663a6","entry":"unravel_index","repo":"lil-lab/phrase_grounding","repo_kind":"official","path":"src/evals/db_utils.py","file_url":"https://github.com/lil-lab/phrase_grounding/blob/HEAD/src/evals/db_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"df66d21a558663a6"}},{"code_sha256_prefix":"110be10502200f87","entry":"get_pretrained_tokenizer","repo":"lil-lab/phrase_grounding","repo_kind":"official","path":"src/datamodules/datamodule_base.py","file_url":"https://github.com/lil-lab/phrase_grounding/blob/HEAD/src/datamodules/datamodule_base.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"110be10502200f87"}},{"code_sha256_prefix":"d590ef655a44ca6b","entry":"summarize_results","repo":"lil-lab/phrase_grounding","repo_kind":"official","path":"src/evals/db_utils.py","file_url":"https://github.com/lil-lab/phrase_grounding/blob/HEAD/src/evals/db_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d590ef655a44ca6b"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}