{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/referring-expression-object-segmentation-with","title":"Referring Expression Object Segmentation with Caption-Aware Consistency","arxiv_id":"1910.04748","date":"2019-10-10","proceeding":null,"authors":["Yi-Wen Chen","Yi-Hsuan Tsai","Tiantian Wang","Yen-Yu Lin","Ming-Hsuan Yang"],"abstract":"Referring expressions are natural language descriptions that identify a particular object within a scene and are widely used in our daily conversations. In this work, we focus on segmenting the object in an image specified by a referring expression. To this end, we propose an end-to-end trainable comprehension network that consists of the language and visual encoders to extract feature representations from both domains. We introduce the spatial-aware dynamic filters to transfer knowledge from text to image, and effectively capture the spatial information of the specified object. To better communicate between the language and visual modules, we employ a caption generation network that takes features shared across both domains as input, and improves both representations via a consistency that enforces the generated sentence to be similar to the given referring expression. We evaluate the proposed framework on two referring expression datasets and show that our method performs favorably against the state-of-the-art algorithms.","url_abs":"https://arxiv.org/abs/1910.04748v1","url_pdf":"https://arxiv.org/pdf/1910.04748v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"referring-expression-object-segmentation-with","repo_url":"https://github.com/wenz116/lang2seg","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"caption-generation","task_name":"Caption Generation"},{"task_slug":"object","task_name":"Object"},{"task_slug":"referring-expression","task_name":"Referring Expression"},{"task_slug":"referring-expression-segmentation","task_name":"Referring Expression Segmentation"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"},{"task_slug":"sentence","task_name":"Sentence"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco","task":"Referring Expression Segmentation","dataset":"RefCoCo val","model":"LANG2SEG","rank_in_archive_order":32,"of":37,"metrics":{"Overall IoU":"58.90"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1910.04748","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1910.04748"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/wenz116/lang2seg","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":2},"by_repo_kind":{"official":{"samples":2,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"d3e019b11b709edb","entry":"xywh_to_xyxy","repo":"wenz116/lang2seg","repo_kind":"official","path":"lib/loaders/caption_loader.py","file_url":"https://github.com/wenz116/lang2seg/blob/HEAD/lib/loaders/caption_loader.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d3e019b11b709edb"}},{"code_sha256_prefix":"a832e5017f504280","entry":"xyxy_to_xywh","repo":"wenz116/lang2seg","repo_kind":"official","path":"lib/loaders/caption_loader.py","file_url":"https://github.com/wenz116/lang2seg/blob/HEAD/lib/loaders/caption_loader.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a832e5017f504280"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}