{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/vlt-vision-language-transformer-and-query","title":"VLT: Vision-Language Transformer and Query Generation for Referring Segmentation","arxiv_id":"2210.15871","date":"2022-10-28","proceeding":null,"authors":["Henghui Ding","Chang Liu","Suchen Wang","Xudong Jiang"],"abstract":"We propose a Vision-Language Transformer (VLT) framework for referring segmentation to facilitate deep interactions among multi-modal information and enhance the holistic understanding to vision-language features. There are different ways to understand the dynamic emphasis of a language expression, especially when interacting with the image. However, the learned queries in existing transformer works are fixed after training, which cannot cope with the randomness and huge diversity of the language expressions. To address this issue, we propose a Query Generation Module, which dynamically produces multiple sets of input-specific queries to represent the diverse comprehensions of language expression. To find the best among these diverse comprehensions, so as to generate a better mask, we propose a Query Balance Module to selectively fuse the corresponding responses of the set of queries. Furthermore, to enhance the model's ability in dealing with diverse language expressions, we consider inter-sample learning to explicitly endow the model with knowledge of understanding different language expressions to the same object. We introduce masked contrastive learning to narrow down the features of different expressions for the same target object while distinguishing the features of different objects. The proposed approach is lightweight and achieves new state-of-the-art referring segmentation results consistently on five datasets.","url_abs":"https://arxiv.org/abs/2210.15871v1","url_pdf":"https://arxiv.org/pdf/2210.15871v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"vlt-vision-language-transformer-and-query","repo_url":"https://github.com/henghuiding/Vision-Language-Transformer","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"tf","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"referring-expression-segmentation","task_name":"Referring Expression Segmentation"},{"task_slug":"referring-video-object-segmentation","task_name":"Referring Video Object Segmentation"},{"task_slug":"video-object-segmentation","task_name":"Video Object Segmentation"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"contrastive-learning","method_name":"Contrastive Learning"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco-5","task":"Referring Expression Segmentation","dataset":"RefCOCO+ test B","model":"VLT","rank_in_archive_order":17,"of":30,"metrics":{"Overall IoU":"56.92"},"uses_additional_data":false},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco-4","task":"Referring Expression Segmentation","dataset":"RefCOCO+ testA","model":"VLT","rank_in_archive_order":18,"of":30,"metrics":{"Overall IoU":"68.43"},"uses_additional_data":false},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco-3","task":"Referring Expression Segmentation","dataset":"RefCOCO+ val","model":"VLT","rank_in_archive_order":21,"of":33,"metrics":{"Overall IoU":"63.53"},"uses_additional_data":false},{"leaderboard":"/sota/referring-expression-segmentation-on-refcocog","task":"Referring Expression Segmentation","dataset":"RefCOCOg-val","model":"VLT (Swin-B)","rank_in_archive_order":18,"of":23,"metrics":{"Overall IoU":"63.49"},"uses_additional_data":false},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco","task":"Referring Expression Segmentation","dataset":"RefCoCo val","model":"VLT","rank_in_archive_order":22,"of":37,"metrics":{"Overall IoU":"72.96"},"uses_additional_data":false},{"leaderboard":"/sota/referring-expression-segmentation-on-refer-1","task":"Referring Expression Segmentation","dataset":"Refer-YouTube-VOS (2021 public validation)","model":"VLT","rank_in_archive_order":20,"of":33,"metrics":{"F":"65.6","J":"61.9","J&F":"63.8"},"uses_additional_data":false},{"leaderboard":"/sota/referring-video-object-segmentation-on-mevis","task":"Referring Video Object Segmentation","dataset":"MeViS","model":"VLT+TC","rank_in_archive_order":11,"of":16,"metrics":{"F":"37.3","J":"33.6","J&F":"35.5"},"uses_additional_data":false},{"leaderboard":"/sota/referring-video-object-segmentation-on-refer","task":"Referring Video Object Segmentation","dataset":"Refer-YouTube-VOS","model":"VLT","rank_in_archive_order":9,"of":18,"metrics":{"F":"65.6","J":"61.9","J&F":"63.8"},"uses_additional_data":true}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2210.15871","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.15871"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/henghuiding/Vision-Language-Transformer","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":6},"by_repo_kind":{"official":{"samples":6,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"47337bf91f204990","entry":"expand_and_tile","repo":"henghuiding/Vision-Language-Transformer","repo_kind":"official","path":"model/utils.py","file_url":"https://github.com/henghuiding/Vision-Language-Transformer/blob/HEAD/model/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"47337bf91f204990"}},{"code_sha256_prefix":"a6d7706f7a9a3eca","entry":"expand_and_tile_1","repo":"henghuiding/Vision-Language-Transformer","repo_kind":"official","path":"model/utils.py","file_url":"https://github.com/henghuiding/Vision-Language-Transformer/blob/HEAD/model/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a6d7706f7a9a3eca"}},{"code_sha256_prefix":"19e932448eddc911","entry":"get_random_data","repo":"henghuiding/Vision-Language-Transformer","repo_kind":"official","path":"loader/loader.py","file_url":"https://github.com/henghuiding/Vision-Language-Transformer/blob/HEAD/loader/loader.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"19e932448eddc911"}},{"code_sha256_prefix":"6f691e355615b0f3","entry":"lr_step_decay","repo":"henghuiding/Vision-Language-Transformer","repo_kind":"official","path":"callbacks/learning_scheduler.py","file_url":"https://github.com/henghuiding/Vision-Language-Transformer/blob/HEAD/callbacks/learning_scheduler.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6f691e355615b0f3"}},{"code_sha256_prefix":"364718e6449b4023","entry":"normalize_by_dim","repo":"henghuiding/Vision-Language-Transformer","repo_kind":"official","path":"model/utils.py","file_url":"https://github.com/henghuiding/Vision-Language-Transformer/blob/HEAD/model/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"364718e6449b4023"}},{"code_sha256_prefix":"00b7240d09897b4f","entry":"qlist_to_vec","repo":"henghuiding/Vision-Language-Transformer","repo_kind":"official","path":"loader/loader.py","file_url":"https://github.com/henghuiding/Vision-Language-Transformer/blob/HEAD/loader/loader.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"00b7240d09897b4f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}