{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/refvos-a-closer-look-at-referring-expressions","title":"RefVOS: A Closer Look at Referring Expressions for Video Object Segmentation","arxiv_id":"2010.00263","date":"2020-10-01","proceeding":null,"authors":["Miriam Bellver","Carles Ventura","Carina Silberer","Ioannis Kazakos","Jordi Torres","Xavier Giro-i-Nieto"],"abstract":"The task of video object segmentation with referring expressions (language-guided VOS) is to, given a linguistic phrase and a video, generate binary masks for the object to which the phrase refers. Our work argues that existing benchmarks used for this task are mainly composed of trivial cases, in which referents can be identified with simple phrases. Our analysis relies on a new categorization of the phrases in the DAVIS-2017 and Actor-Action datasets into trivial and non-trivial REs, with the non-trivial REs annotated with seven RE semantic categories. We leverage this data to analyze the results of RefVOS, a novel neural network that obtains competitive results for the task of language-guided image segmentation and state of the art results for language-guided VOS. Our study indicates that the major challenges for the task are related to understanding motion and static actions.","url_abs":"https://arxiv.org/abs/2010.00263v1","url_pdf":"https://arxiv.org/pdf/2010.00263v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"refvos-a-closer-look-at-referring-expressions","repo_url":"https://github.com/miriambellver/refvos","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"refvos-a-closer-look-at-referring-expressions","repo_url":"https://github.com/imatge-upc/refvos","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"image-segmentation","task_name":"Image Segmentation"},{"task_slug":"referring-expression-segmentation","task_name":"Referring Expression Segmentation"},{"task_slug":"segmentation","task_name":"Segmentation"},{"task_slug":"video-object-segmentation","task_name":"Video Object Segmentation"}],"methods":[{"method_slug":"1x1-convolution","method_name":"1x1 Convolution"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"convolution","method_name":"Convolution"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dilated-convolution","method_name":"Dilated Convolution"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"grouped-convolution","method_name":"Grouped Convolution"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"multiscale-dilated-convolution-block","method_name":"Multiscale Dilated Convolution Block"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"vos","method_name":"VOS"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[{"slug":"a2d-referring-expressions","name":"A2Dre","full_name":"Subset of A2D Sentences which are not trivial"},{"slug":"a2dre","name":"A2Dre+","full_name":"Extension of A2D sentences where trivial cases where filtered"}],"methods_introduced":[],"results":[{"leaderboard":"/sota/referring-expression-segmentation-on-a2d","task":"Referring Expression Segmentation","dataset":"A2D Sentences","model":"RefVOS","rank_in_archive_order":27,"of":27,"metrics":{"IoU mean":"0.599","IoU overall":"0.599","Precision@0.5":"0.495","Precision@0.9":"0.064"},"uses_additional_data":false},{"leaderboard":"/sota/referring-expression-segmentation-on-a2dre","task":"Referring Expression Segmentation","dataset":"A2Dre test","model":"RefVos","rank_in_archive_order":1,"of":1,"metrics":{"Mean IoU":"33.2","Overall IoU":"47.5"},"uses_additional_data":false},{"leaderboard":"/sota/referring-expression-segmentation-on-davis","task":"Referring Expression Segmentation","dataset":"DAVIS 2017 (val)","model":"RefVOS","rank_in_archive_order":14,"of":18,"metrics":{"J&F 1st frame":"44.5","J&F Full video":"45.1"},"uses_additional_data":false},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco-5","task":"Referring Expression Segmentation","dataset":"RefCOCO+ test B","model":"RefVOS with BERT + MLM loss","rank_in_archive_order":28,"of":30,"metrics":{"Overall IoU":"36.17"},"uses_additional_data":false},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco-4","task":"Referring Expression Segmentation","dataset":"RefCOCO+ testA","model":"RefVOS with BERT + MLM Loss","rank_in_archive_order":28,"of":30,"metrics":{"Overall IoU":"49.73"},"uses_additional_data":false},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco-3","task":"Referring Expression Segmentation","dataset":"RefCOCO+ val","model":"RefVOS with BERT + MLM loss","rank_in_archive_order":31,"of":33,"metrics":{"Overall IoU":"44.71"},"uses_additional_data":false},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco","task":"Referring Expression Segmentation","dataset":"RefCoCo val","model":"RefVOS with BERT + MLM loss","rank_in_archive_order":31,"of":37,"metrics":{"Overall IoU":"59.45"},"uses_additional_data":false},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco","task":"Referring Expression Segmentation","dataset":"RefCoCo val","model":"RefVOS with BERT Pre-train","rank_in_archive_order":33,"of":37,"metrics":{"Overall IoU":"58.65"},"uses_additional_data":false}],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2010.00263","mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}