{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/disambiguating-reference-in-visually-grounded","title":"Disambiguating Reference in Visually Grounded Dialogues through Joint Modeling of Textual and Multimodal Semantic Structures","arxiv_id":"2505.11726","date":"2025-05-16","proceeding":null,"authors":["Shun Inadumi","Nobuhiro Ueda","Koichiro Yoshino"],"abstract":"Multimodal reference resolution, including phrase grounding, aims to understand the semantic relations between mentions and real-world objects. Phrase grounding between images and their captions is a well-established task. In contrast, for real-world applications, it is essential to integrate textual and multimodal reference resolution to unravel the reference relations within dialogue, especially in handling ambiguities caused by pronouns and ellipses. This paper presents a framework that unifies textual and multimodal reference resolution by mapping mention embeddings to object embeddings and selecting mentions or objects based on their similarity. Our experiments show that learning textual reference resolution, such as coreference resolution and predicate-argument structure analysis, positively affects performance in multimodal reference resolution. In particular, our model with coreference resolution performs better in pronoun phrase grounding than representative models for this task, MDETR and GLIP. Our qualitative analysis demonstrates that incorporating textual reference relations strengthens the confidence scores between mentions, including pronouns and predicates, and objects, which can reduce the ambiguities that arise in visually grounded dialogues.","url_abs":"https://arxiv.org/abs/2505.11726v1","url_pdf":"https://arxiv.org/pdf/2505.11726v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"disambiguating-reference-in-visually-grounded","repo_url":"https://github.com/sinadumi/mmrr","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"coreference-resolution","task_name":"Coreference Resolution"},{"task_slug":"phrase-grounding","task_name":"Phrase Grounding"},{"task_slug":"coreference-resolution-1","task_name":"coreference-resolution"}],"methods":[{"method_slug":"mdetr","method_name":"MDETR"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2505.11726","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.11726"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/ashkamath/mdetr","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_violates":1,"ran_draft_wrong":1,"unverified":4},"by_repo_kind":{"found_in_text":{"samples":6,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"ac8fe530cdad4d8c","entry":"dice_loss","repo":"ashkamath/mdetr","repo_kind":"found_in_text","path":"models/segmentation.py","file_url":"https://github.com/ashkamath/mdetr/blob/HEAD/models/segmentation.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ac8fe530cdad4d8c"}},{"code_sha256_prefix":"5c0711aada67957e","entry":"sigmoid_focal_loss","repo":"ashkamath/mdetr","repo_kind":"found_in_text","path":"models/segmentation.py","file_url":"https://github.com/ashkamath/mdetr/blob/HEAD/models/segmentation.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5c0711aada67957e"}},{"code_sha256_prefix":"cea3f92defccfb9c","entry":"build_position_encoding","repo":"ashkamath/mdetr","repo_kind":"found_in_text","path":"models/position_encoding.py","file_url":"https://github.com/ashkamath/mdetr/blob/HEAD/models/position_encoding.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cea3f92defccfb9c"}},{"code_sha256_prefix":"0796c9e7b3ba5e23","entry":"build_transformer","repo":"ashkamath/mdetr","repo_kind":"found_in_text","path":"models/transformer.py","file_url":"https://github.com/ashkamath/mdetr/blob/HEAD/models/transformer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0796c9e7b3ba5e23"}},{"code_sha256_prefix":"d71abefc7fbfd28f","entry":"get_init_file","repo":"ashkamath/mdetr","repo_kind":"found_in_text","path":"run_with_submitit.py","file_url":"https://github.com/ashkamath/mdetr/blob/HEAD/run_with_submitit.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d71abefc7fbfd28f"}},{"code_sha256_prefix":"84691ff17ae1504d","entry":"get_shared_folder","repo":"ashkamath/mdetr","repo_kind":"found_in_text","path":"run_with_submitit.py","file_url":"https://github.com/ashkamath/mdetr/blob/HEAD/run_with_submitit.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"84691ff17ae1504d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}