{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/composing-object-relations-and-attributes-for-1","title":"Composing Object Relations and Attributes for Image-Text Matching","arxiv_id":"2406.11820","date":"2024-06-17","proceeding":"CVPR 2024 1","authors":["Khoi Pham","Chuong Huynh","Ser-Nam Lim","Abhinav Shrivastava"],"abstract":"We study the visual semantic embedding problem for image-text matching. Most existing work utilizes a tailored cross-attention mechanism to perform local alignment across the two image and text modalities. This is computationally expensive, even though it is more powerful than the unimodal dual-encoder approach. This work introduces a dual-encoder image-text matching model, leveraging a scene graph to represent captions with nodes for objects and attributes interconnected by relational edges. Utilizing a graph attention network, our model efficiently encodes object-attribute and object-object semantic relations, resulting in a robust and fast-performing system. Representing caption as a scene graph offers the ability to utilize the strong relational inductive bias of graph neural networks to learn object-attribute and object-object relations effectively. To train the model, we propose losses that align the image and caption both at the holistic level (image-caption) and the local level (image-object entity), which we show is key to the success of the model. Our model is termed Composition model for Object Relations and Attributes, CORA. Experimental results on two prominent image-text retrieval benchmarks, Flickr30K and MSCOCO, demonstrate that CORA outperforms existing state-of-the-art computationally expensive cross-attention methods regarding recall score while achieving fast computation speed of the dual encoder.","url_abs":"https://arxiv.org/abs/2406.11820v1","url_pdf":"https://arxiv.org/pdf/2406.11820v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"composing-object-relations-and-attributes-for-1","repo_url":"https://github.com/vkhoi/cora_cvpr24","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"attribute","task_name":"Attribute"},{"task_slug":"graph-attention","task_name":"Graph Attention"},{"task_slug":"image-text-retrieval","task_name":"Image-text Retrieval"},{"task_slug":"image-text-matching","task_name":"Image-text matching"},{"task_slug":"inductive-bias","task_name":"Inductive Bias"},{"task_slug":"object","task_name":"Object"},{"task_slug":"text-matching","task_name":"Text Matching"},{"task_slug":"text-retrieval","task_name":"Text Retrieval"}],"methods":[{"method_slug":"align","method_name":"ALIGN"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"speed","method_name":"SPEED"},{"method_slug":"softmax","method_name":"Softmax"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2406.11820","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.11820"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/vkhoi/cora_cvpr24","reach":null}],"summary":{"ran":1,"ran_draft_wrong":1,"ran_fixture":2,"unverified":1},"by_repo_kind":{"official":{"samples":5,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":5,"samples":[{"code_sha256_prefix":"9acd7ed5fa371c90","entry":"GatherLayer","repo":"vkhoi/cora_cvpr24","repo_kind":"official","path":"models/cora_model.py","file_url":"https://github.com/vkhoi/cora_cvpr24/blob/HEAD/models/cora_model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9acd7ed5fa371c90"}},{"code_sha256_prefix":"f88c7aabf2f29a06","entry":"image_caption_triplet_loss","repo":"vkhoi/cora_cvpr24","repo_kind":"official","path":"models/cora_model.py","file_url":"https://github.com/vkhoi/cora_cvpr24/blob/HEAD/models/cora_model.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f88c7aabf2f29a06"}},{"code_sha256_prefix":"e1d390f8d010d156","entry":"supcon_joint_caption_entity_loss","repo":"vkhoi/cora_cvpr24","repo_kind":"official","path":"models/cora_model.py","file_url":"https://github.com/vkhoi/cora_cvpr24/blob/HEAD/models/cora_model.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e1d390f8d010d156"}},{"code_sha256_prefix":"3003e74a6662d0cb","entry":"triplet_specificity_loss","repo":"vkhoi/cora_cvpr24","repo_kind":"official","path":"models/cora_model.py","file_url":"https://github.com/vkhoi/cora_cvpr24/blob/HEAD/models/cora_model.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3003e74a6662d0cb"}},{"code_sha256_prefix":"6eb0eeba741b84e7","entry":"CoraModel","repo":"vkhoi/cora_cvpr24","repo_kind":"official","path":"models/cora_model.py","file_url":"https://github.com/vkhoi/cora_cvpr24/blob/HEAD/models/cora_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6eb0eeba741b84e7"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}