{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/mask-grounding-for-referring-image","title":"Mask Grounding for Referring Image Segmentation","arxiv_id":"2312.12198","date":"2023-12-19","proceeding":"CVPR 2024 1","authors":["Yong Xien Chng","Henry Zheng","Yizeng Han","Xuchong Qiu","Gao Huang"],"abstract":"Referring Image Segmentation (RIS) is a challenging task that requires an algorithm to segment objects referred by free-form language expressions. Despite significant progress in recent years, most state-of-the-art (SOTA) methods still suffer from considerable language-image modality gap at the pixel and word level. These methods generally 1) rely on sentence-level language features for language-image alignment and 2) lack explicit training supervision for fine-grained visual grounding. Consequently, they exhibit weak object-level correspondence between visual and language features. Without well-grounded features, prior methods struggle to understand complex expressions that require strong reasoning over relationships among multiple objects, especially when dealing with rarely used or ambiguous clauses. To tackle this challenge, we introduce a novel Mask Grounding auxiliary task that significantly improves visual grounding within language features, by explicitly teaching the model to learn fine-grained correspondence between masked textual tokens and their matching visual objects. Mask Grounding can be directly used on prior RIS methods and consistently bring improvements. Furthermore, to holistically address the modality gap, we also design a cross-modal alignment loss and an accompanying alignment module. These additions work synergistically with Mask Grounding. With all these techniques, our comprehensive approach culminates in MagNet (Mask-grounded Network), an architecture that significantly outperforms prior arts on three key benchmarks (RefCOCO, RefCOCO+ and G-Ref), demonstrating our method's effectiveness in addressing current limitations of RIS algorithms. Our code and pre-trained weights will be released.","url_abs":"https://arxiv.org/abs/2312.12198v2","url_pdf":"https://arxiv.org/pdf/2312.12198v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"mask-grounding-for-referring-image","repo_url":"https://github.com/yxchng/mask-grounding","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"AGPL-3.0"}}],"tasks":[{"task_slug":"image-segmentation","task_name":"Image Segmentation"},{"task_slug":"referring-expression-segmentation","task_name":"Referring Expression Segmentation"},{"task_slug":"segmentation","task_name":"Segmentation"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"},{"task_slug":"sentence","task_name":"Sentence"},{"task_slug":"visual-grounding","task_name":"Visual Grounding"},{"task_slug":"cross-modal-alignment","task_name":"cross-modal alignment"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco-8","task":"Referring Expression Segmentation","dataset":"RefCOCO testA","model":"MagNet","rank_in_archive_order":10,"of":13,"metrics":{"Overall IoU":"78.24"},"uses_additional_data":false},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco-9","task":"Referring Expression Segmentation","dataset":"RefCOCO testB","model":"MagNet","rank_in_archive_order":10,"of":13,"metrics":{"Overall IoU":"71.05"},"uses_additional_data":false},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco-5","task":"Referring Expression Segmentation","dataset":"RefCOCO+ test B","model":"MagNet","rank_in_archive_order":15,"of":30,"metrics":{"Overall IoU":"58.14"},"uses_additional_data":false},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco-4","task":"Referring Expression Segmentation","dataset":"RefCOCO+ testA","model":"MagNet","rank_in_archive_order":16,"of":30,"metrics":{"Overall IoU":"71.32"},"uses_additional_data":false},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco-3","task":"Referring Expression Segmentation","dataset":"RefCOCO+ val","model":"MagNet","rank_in_archive_order":19,"of":33,"metrics":{"Overall IoU":"66.16"},"uses_additional_data":false},{"leaderboard":"/sota/referring-expression-segmentation-on-refcocog-1","task":"Referring Expression Segmentation","dataset":"RefCOCOg-test","model":"MagNet","rank_in_archive_order":14,"of":18,"metrics":{"Overall IoU":"66.03"},"uses_additional_data":false},{"leaderboard":"/sota/referring-expression-segmentation-on-refcocog","task":"Referring Expression Segmentation","dataset":"RefCOCOg-val","model":"MagNet","rank_in_archive_order":16,"of":23,"metrics":{"Overall IoU":"65.36"},"uses_additional_data":false},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco","task":"Referring Expression Segmentation","dataset":"RefCoCo val","model":"MagNet","rank_in_archive_order":18,"of":37,"metrics":{"Overall IoU":"75.24"},"uses_additional_data":false}],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2312.12198","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.12198"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yxchng/mask-grounding","reach":{"status":"ok","spdx":"AGPL-3.0"}}],"summary":{"ran_draft_wrong":4,"ran_fixture":1,"ran":5,"unverified":3},"by_repo_kind":{"official":{"samples":13,"ran":10,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":13,"samples":[{"code_sha256_prefix":"0f786c407fb1ee4c","entry":"swish","repo":"yxchng/mask-grounding","repo_kind":"official","path":"bert/activations.py","file_url":"https://github.com/yxchng/mask-grounding/blob/HEAD/bert/activations.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":2,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"0f786c407fb1ee4c"}},{"code_sha256_prefix":"144d10b49baeb8a6","entry":"window_partition","repo":"yxchng/mask-grounding","repo_kind":"official","path":"lib/backbone.py","file_url":"https://github.com/yxchng/mask-grounding/blob/HEAD/lib/backbone.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":2,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"144d10b49baeb8a6"}},{"code_sha256_prefix":"b239247b87d4f3dc","entry":"find_pruneable_heads_and_indices","repo":"yxchng/mask-grounding","repo_kind":"official","path":"bert/modeling_utils.py","file_url":"https://github.com/yxchng/mask-grounding/blob/HEAD/bert/modeling_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"b239247b87d4f3dc"}},{"code_sha256_prefix":"a4475703ff58ecf9","entry":"gelu_fast","repo":"yxchng/mask-grounding","repo_kind":"official","path":"bert/activations.py","file_url":"https://github.com/yxchng/mask-grounding/blob/HEAD/bert/activations.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"a4475703ff58ecf9"}},{"code_sha256_prefix":"dc9ffc0f29e7fa3d","entry":"gelu_new","repo":"yxchng/mask-grounding","repo_kind":"official","path":"bert/activations.py","file_url":"https://github.com/yxchng/mask-grounding/blob/HEAD/bert/activations.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"dc9ffc0f29e7fa3d"}},{"code_sha256_prefix":"22b41155e451683b","entry":"mish","repo":"yxchng/mask-grounding","repo_kind":"official","path":"bert/modeling_bert.py","file_url":"https://github.com/yxchng/mask-grounding/blob/HEAD/bert/modeling_bert.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"22b41155e451683b"}},{"code_sha256_prefix":"452475fb6da97402","entry":"pad_if_smaller","repo":"yxchng/mask-grounding","repo_kind":"official","path":"transforms.py","file_url":"https://github.com/yxchng/mask-grounding/blob/HEAD/transforms.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"452475fb6da97402"}},{"code_sha256_prefix":"b3ee64494a3b6d38","entry":"prune_conv1d_layer","repo":"yxchng/mask-grounding","repo_kind":"official","path":"bert/modeling_utils.py","file_url":"https://github.com/yxchng/mask-grounding/blob/HEAD/bert/modeling_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"b3ee64494a3b6d38"}},{"code_sha256_prefix":"a37615738b3ddcd3","entry":"prune_linear_layer","repo":"yxchng/mask-grounding","repo_kind":"official","path":"bert/modeling_utils.py","file_url":"https://github.com/yxchng/mask-grounding/blob/HEAD/bert/modeling_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"a37615738b3ddcd3"}},{"code_sha256_prefix":"61bf152e6a42a184","entry":"window_reverse","repo":"yxchng/mask-grounding","repo_kind":"official","path":"lib/backbone.py","file_url":"https://github.com/yxchng/mask-grounding/blob/HEAD/lib/backbone.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"61bf152e6a42a184"}},{"code_sha256_prefix":"67fee1d4b5d4493d","entry":"lavt","repo":"yxchng/mask-grounding","repo_kind":"official","path":"lib/multimodal_segmentation.py","file_url":"https://github.com/yxchng/mask-grounding/blob/HEAD/lib/multimodal_segmentation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"67fee1d4b5d4493d"}},{"code_sha256_prefix":"2944cc78660b63ff","entry":"lavt_one","repo":"yxchng/mask-grounding","repo_kind":"official","path":"lib/multimodal_segmentation.py","file_url":"https://github.com/yxchng/mask-grounding/blob/HEAD/lib/multimodal_segmentation.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"2944cc78660b63ff"}},{"code_sha256_prefix":"fb9398de2966d6de","entry":"load_tf_weights_in_bert","repo":"yxchng/mask-grounding","repo_kind":"official","path":"bert/modeling_bert.py","file_url":"https://github.com/yxchng/mask-grounding/blob/HEAD/bert/modeling_bert.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"fb9398de2966d6de"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}