{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/decoupling-static-and-hierarchical-motion","title":"Decoupling Static and Hierarchical Motion Perception for Referring Video Segmentation","arxiv_id":"2404.03645","date":"2024-04-04","proceeding":"CVPR 2024 1","authors":["Shuting He","Henghui Ding"],"abstract":"Referring video segmentation relies on natural language expressions to identify and segment objects, often emphasizing motion clues. Previous works treat a sentence as a whole and directly perform identification at the video-level, mixing up static image-level cues with temporal motion cues. However, image-level features cannot well comprehend motion cues in sentences, and static cues are not crucial for temporal perception. In fact, static cues can sometimes interfere with temporal perception by overshadowing motion cues. In this work, we propose to decouple video-level referring expression understanding into static and motion perception, with a specific emphasis on enhancing temporal comprehension. Firstly, we introduce an expression-decoupling module to make static cues and motion cues perform their distinct role, alleviating the issue of sentence embeddings overlooking motion cues. Secondly, we propose a hierarchical motion perception module to capture temporal information effectively across varying timescales. Furthermore, we employ contrastive learning to distinguish the motions of visually similar objects. These contributions yield state-of-the-art performance across five datasets, including a remarkable $\\textbf{9.2%}$ $\\mathcal{J\\&F}$ improvement on the challenging $\\textbf{MeViS}$ dataset. Code is available at https://github.com/heshuting555/DsHmp.","url_abs":"https://arxiv.org/abs/2404.03645v1","url_pdf":"https://arxiv.org/pdf/2404.03645v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"decoupling-static-and-hierarchical-motion","repo_url":"https://github.com/heshuting555/dshmp","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"contrastive-learning","task_name":"Contrastive Learning"},{"task_slug":"referring-expression","task_name":"Referring Expression"},{"task_slug":"referring-expression-segmentation","task_name":"Referring Expression Segmentation"},{"task_slug":"referring-video-object-segmentation","task_name":"Referring Video Object Segmentation"},{"task_slug":"sentence","task_name":"Sentence"},{"task_slug":"sentence-embeddings","task_name":"Sentence Embeddings"},{"task_slug":"video-segmentation","task_name":"Video Segmentation"},{"task_slug":"video-semantic-segmentation","task_name":"Video Semantic Segmentation"}],"methods":[{"method_slug":"contrastive-learning","method_name":"Contrastive Learning"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/referring-expression-segmentation-on-refer-1","task":"Referring Expression Segmentation","dataset":"Refer-YouTube-VOS (2021 public validation)","model":"DsHmp (Video-Swin-Base)","rank_in_archive_order":11,"of":33,"metrics":{"F":"69.1","J":"65","J&F":"67.1"},"uses_additional_data":false},{"leaderboard":"/sota/referring-video-object-segmentation-on-mevis","task":"Referring Video Object Segmentation","dataset":"MeViS","model":"DsHmp","rank_in_archive_order":8,"of":16,"metrics":{"F":"49.8","J":"43","J&F":"46.4"},"uses_additional_data":false},{"leaderboard":"/sota/referring-video-object-segmentation-on-ref","task":"Referring Video Object Segmentation","dataset":"Ref-DAVIS17","model":"DsHmp","rank_in_archive_order":5,"of":11,"metrics":{"F":"68.1","J":"61.7","J&F":"64.9"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2404.03645","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.03645"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/heshuting555/dshmp","reach":{"status":"ok"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/heshuting555/DsHmp","reach":{"status":"ok"}}],"summary":{"ran":2,"unverified":5},"by_repo_kind":{"official":{"samples":7,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":7,"samples":[{"code_sha256_prefix":"9835cf213d40b39e","entry":"default_argument_parser","repo":"heshuting555/DsHmp","repo_kind":"official","path":"dshmp/engine/defaults.py","file_url":"https://github.com/heshuting555/DsHmp/blob/HEAD/dshmp/engine/defaults.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9835cf213d40b39e"}},{"code_sha256_prefix":"596ba9632eba6fdb","entry":"multi_pos_cross_entropy","repo":"heshuting555/DsHmp","repo_kind":"official","path":"dshmp/modeling/contrastive_loss.py","file_url":"https://github.com/heshuting555/DsHmp/blob/HEAD/dshmp/modeling/contrastive_loss.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"596ba9632eba6fdb"}},{"code_sha256_prefix":"bc2cb481a75c370d","entry":"batch_dice_loss","repo":"heshuting555/DsHmp","repo_kind":"official","path":"dshmp/modeling/vita_matcher.py","file_url":"https://github.com/heshuting555/DsHmp/blob/HEAD/dshmp/modeling/vita_matcher.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bc2cb481a75c370d"}},{"code_sha256_prefix":"1edd24985036b0bf","entry":"batch_sigmoid_ce_loss","repo":"heshuting555/DsHmp","repo_kind":"official","path":"dshmp/modeling/vita_matcher.py","file_url":"https://github.com/heshuting555/DsHmp/blob/HEAD/dshmp/modeling/vita_matcher.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1edd24985036b0bf"}},{"code_sha256_prefix":"2dcb8123d89bb1ff","entry":"calculate_uncertainty","repo":"heshuting555/DsHmp","repo_kind":"official","path":"dshmp/modeling/vita_criterion.py","file_url":"https://github.com/heshuting555/DsHmp/blob/HEAD/dshmp/modeling/vita_criterion.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2dcb8123d89bb1ff"}},{"code_sha256_prefix":"89f75e54ff128be0","entry":"dice_loss","repo":"heshuting555/DsHmp","repo_kind":"official","path":"dshmp/modeling/vita_criterion.py","file_url":"https://github.com/heshuting555/DsHmp/blob/HEAD/dshmp/modeling/vita_criterion.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"89f75e54ff128be0"}},{"code_sha256_prefix":"d0c61e8dba511aa3","entry":"sigmoid_ce_loss","repo":"heshuting555/DsHmp","repo_kind":"official","path":"dshmp/modeling/vita_criterion.py","file_url":"https://github.com/heshuting555/DsHmp/blob/HEAD/dshmp/modeling/vita_criterion.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d0c61e8dba511aa3"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}