{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/hyperseg-towards-universal-visual","title":"HyperSeg: Towards Universal Visual Segmentation with Large Language Model","arxiv_id":"2411.17606","date":"2024-11-26","proceeding":null,"authors":["Cong Wei","Yujie Zhong","Haoxian Tan","Yong liu","Zheng Zhao","Jie Hu","Yujiu Yang"],"abstract":"This paper aims to address universal segmentation for image and video perception with the strong reasoning ability empowered by Visual Large Language Models (VLLMs). Despite significant progress in current unified segmentation methods, limitations in adaptation to both image and video scenarios, as well as the complex reasoning segmentation, make it difficult for them to handle various challenging instructions and achieve an accurate understanding of fine-grained vision-language correlations. We propose HyperSeg, the first VLLM-based universal segmentation model for pixel-level image and video perception, encompassing generic segmentation tasks and more complex reasoning perception tasks requiring powerful reasoning abilities and world knowledge. Besides, to fully leverage the recognition capabilities of VLLMs and the fine-grained visual information, HyperSeg incorporates hybrid entity recognition and fine-grained visual perceiver modules for various segmentation tasks. Combined with the temporal adapter, HyperSeg achieves a comprehensive understanding of temporal information. Experimental results validate the effectiveness of our insights in resolving universal image and video segmentation tasks, including the more complex reasoning perception tasks. Our code is available.","url_abs":"https://arxiv.org/abs/2411.17606v2","url_pdf":"https://arxiv.org/pdf/2411.17606v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"hyperseg-towards-universal-visual","repo_url":"https://github.com/congvvc/HyperSeg","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"large-language-model","task_name":"Large Language Model"},{"task_slug":"open-vocabulary-semantic-segmentation","task_name":"Open Vocabulary Semantic Segmentation"},{"task_slug":"panoptic-segmentation","task_name":"Panoptic Segmentation"},{"task_slug":"reasoning-segmentation","task_name":"Reasoning Segmentation"},{"task_slug":"reasoning-video-object-segmentation","task_name":"Reasoning Video Object Segmentation"},{"task_slug":"referring-expression-segmentation","task_name":"Referring Expression Segmentation"},{"task_slug":"referring-video-object-segmentation","task_name":"Referring Video Object Segmentation"},{"task_slug":"segmentation","task_name":"Segmentation"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"},{"task_slug":"universal-segmentation","task_name":"Universal Segmentation"},{"task_slug":"video-object-segmentation","task_name":"Video Object Segmentation"},{"task_slug":"world-knowledge","task_name":"World Knowledge"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/open-vocabulary-semantic-segmentation-on-1","task":"Open Vocabulary Semantic Segmentation","dataset":"PASCAL Context-59","model":"HyperSeg","rank_in_archive_order":1,"of":24,"metrics":{"mIoU":"64.6"},"uses_additional_data":true},{"leaderboard":"/sota/open-vocabulary-semantic-segmentation-on-5","task":"Open Vocabulary Semantic Segmentation","dataset":"PascalVOC-20","model":"HyperSeg","rank_in_archive_order":12,"of":20,"metrics":{"mIoU":"92.1"},"uses_additional_data":true},{"leaderboard":"/sota/panoptic-segmentation-on-coco-minival","task":"Panoptic Segmentation","dataset":"COCO minival","model":"HyperSeg (Swin-B)","rank_in_archive_order":1,"of":31,"metrics":{"PQ":"61.2"},"uses_additional_data":true},{"leaderboard":"/sota/referring-expression-segmentation-on-davis","task":"Referring Expression Segmentation","dataset":"DAVIS 2017 (val)","model":"HyperSeg","rank_in_archive_order":2,"of":18,"metrics":{"J&F 1st frame":"71.2"},"uses_additional_data":true},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco-8","task":"Referring Expression Segmentation","dataset":"RefCOCO testA","model":"HyperSeg","rank_in_archive_order":2,"of":13,"metrics":{"Overall IoU":"85.7"},"uses_additional_data":true},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco-9","task":"Referring Expression Segmentation","dataset":"RefCOCO testB","model":"HyperSeg","rank_in_archive_order":1,"of":13,"metrics":{"Overall IoU":"83.4"},"uses_additional_data":true},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco-5","task":"Referring Expression Segmentation","dataset":"RefCOCO+ test B","model":"HyperSeg","rank_in_archive_order":2,"of":30,"metrics":{"Overall IoU":"75.2"},"uses_additional_data":true},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco-4","task":"Referring Expression Segmentation","dataset":"RefCOCO+ testA","model":"HyperSeg","rank_in_archive_order":1,"of":30,"metrics":{"Overall IoU":"83.5"},"uses_additional_data":true},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco-3","task":"Referring Expression Segmentation","dataset":"RefCOCO+ val","model":"HyperSeg","rank_in_archive_order":3,"of":33,"metrics":{"Overall IoU":"79.0"},"uses_additional_data":true},{"leaderboard":"/sota/referring-expression-segmentation-on-refcocog-1","task":"Referring Expression Segmentation","dataset":"RefCOCOg-test","model":"HyperSeg","rank_in_archive_order":4,"of":18,"metrics":{"Overall IoU":"78.9"},"uses_additional_data":true},{"leaderboard":"/sota/referring-expression-segmentation-on-refcocog","task":"Referring Expression Segmentation","dataset":"RefCOCOg-val","model":"HyperSeg","rank_in_archive_order":2,"of":23,"metrics":{"Overall IoU":"79.4"},"uses_additional_data":true},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco","task":"Referring Expression Segmentation","dataset":"RefCoCo val","model":"HyperSeg","rank_in_archive_order":2,"of":37,"metrics":{"Overall IoU":"84.8"},"uses_additional_data":true},{"leaderboard":"/sota/referring-video-object-segmentation-on-refer","task":"Referring Video Object Segmentation","dataset":"Refer-YouTube-VOS","model":"HyperSeg","rank_in_archive_order":3,"of":18,"metrics":{"J&F":"68.5"},"uses_additional_data":true},{"leaderboard":"/sota/semantic-segmentation-on-coco-1","task":"Semantic Segmentation","dataset":"COCO (Common Objects in Context)","model":"HyperSeg","rank_in_archive_order":1,"of":9,"metrics":{"mIoU":"77.2"},"uses_additional_data":true}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2411.17606","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2411.17606"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/congvvc/HyperSeg","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_fixture":1,"ran_draft_wrong":3,"ran_honours":1,"ran_violates":1,"ran":1,"unverified":10},"by_repo_kind":{"official":{"samples":17,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"144d10b49baeb8a6","entry":"window_partition","repo":"congvvc/HyperSeg","repo_kind":"official","path":"hyperseg/model/multimodal_encoder/swin_trans.py","file_url":"https://github.com/congvvc/HyperSeg/blob/HEAD/hyperseg/model/multimodal_encoder/swin_trans.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":2,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"144d10b49baeb8a6"}},{"code_sha256_prefix":"4d75570b4e91d0b4","entry":"FeedForward","repo":"congvvc/HyperSeg","repo_kind":"official","path":"hyperseg/model/language_model/perceiver.py","file_url":"https://github.com/congvvc/HyperSeg/blob/HEAD/hyperseg/model/language_model/perceiver.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4d75570b4e91d0b4"}},{"code_sha256_prefix":"e0a06ded5d4f6c3c","entry":"box_cxcywh_to_xyxy","repo":"congvvc/HyperSeg","repo_kind":"official","path":"hyperseg/model/tracker/box_ops.py","file_url":"https://github.com/congvvc/HyperSeg/blob/HEAD/hyperseg/model/tracker/box_ops.py","link_basis":"plan_row","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e0a06ded5d4f6c3c"}},{"code_sha256_prefix":"aa5486a3650902d8","entry":"exists","repo":"congvvc/HyperSeg","repo_kind":"official","path":"hyperseg/model/language_model/perceiver.py","file_url":"https://github.com/congvvc/HyperSeg/blob/HEAD/hyperseg/model/language_model/perceiver.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"aa5486a3650902d8"}},{"code_sha256_prefix":"592b3c1a88f93d7c","entry":"expand2square","repo":"congvvc/HyperSeg","repo_kind":"official","path":"hyperseg/model/mipha/mm_utils.py","file_url":"https://github.com/congvvc/HyperSeg/blob/HEAD/hyperseg/model/mipha/mm_utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"592b3c1a88f93d7c"}},{"code_sha256_prefix":"c3ee9d07c900dd55","entry":"load_image_from_base64","repo":"congvvc/HyperSeg","repo_kind":"official","path":"hyperseg/model/mipha/mm_utils.py","file_url":"https://github.com/congvvc/HyperSeg/blob/HEAD/hyperseg/model/mipha/mm_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c3ee9d07c900dd55"}},{"code_sha256_prefix":"61bf152e6a42a184","entry":"window_reverse","repo":"congvvc/HyperSeg","repo_kind":"official","path":"hyperseg/model/multimodal_encoder/swin_trans.py","file_url":"https://github.com/congvvc/HyperSeg/blob/HEAD/hyperseg/model/multimodal_encoder/swin_trans.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"61bf152e6a42a184"}},{"code_sha256_prefix":"40cf72d9c04c7f63","entry":"build_logger","repo":"congvvc/HyperSeg","repo_kind":"official","path":"hyperseg/model/mipha/utils.py","file_url":"https://github.com/congvvc/HyperSeg/blob/HEAD/hyperseg/model/mipha/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"40cf72d9c04c7f63"}},{"code_sha256_prefix":"670eda77b277eb8a","entry":"build_swin_t","repo":"congvvc/HyperSeg","repo_kind":"official","path":"hyperseg/model/multimodal_encoder/swin_trans.py","file_url":"https://github.com/congvvc/HyperSeg/blob/HEAD/hyperseg/model/multimodal_encoder/swin_trans.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"670eda77b277eb8a"}},{"code_sha256_prefix":"bbe128f6633ed802","entry":"check_shape","repo":"congvvc/HyperSeg","repo_kind":"official","path":"hyperseg/model/language_model/einops_exts.py","file_url":"https://github.com/congvvc/HyperSeg/blob/HEAD/hyperseg/model/language_model/einops_exts.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bbe128f6633ed802"}},{"code_sha256_prefix":"8bbf355ce170aeb9","entry":"enhance_with_circles","repo":"congvvc/HyperSeg","repo_kind":"official","path":"hyperseg/model/datasets_mapper/coco_instance_mapper.py","file_url":"https://github.com/congvvc/HyperSeg/blob/HEAD/hyperseg/model/datasets_mapper/coco_instance_mapper.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8bbf355ce170aeb9"}},{"code_sha256_prefix":"ee5e474262386435","entry":"filter_empty_instances_soft","repo":"congvvc/HyperSeg","repo_kind":"official","path":"hyperseg/model/datasets_mapper/coco_instance_mapper.py","file_url":"https://github.com/congvvc/HyperSeg/blob/HEAD/hyperseg/model/datasets_mapper/coco_instance_mapper.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ee5e474262386435"}},{"code_sha256_prefix":"b1aa0c20f4bb79af","entry":"get_mask_box_from_json","repo":"congvvc/HyperSeg","repo_kind":"official","path":"hyperseg/model/tracker/box_ops.py","file_url":"https://github.com/congvvc/HyperSeg/blob/HEAD/hyperseg/model/tracker/box_ops.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b1aa0c20f4bb79af"}},{"code_sha256_prefix":"5f4db0dd94dcdfcb","entry":"get_mask_from_json","repo":"congvvc/HyperSeg","repo_kind":"official","path":"hyperseg/model/tracker/box_ops.py","file_url":"https://github.com/congvvc/HyperSeg/blob/HEAD/hyperseg/model/tracker/box_ops.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5f4db0dd94dcdfcb"}},{"code_sha256_prefix":"37899f22fb191b37","entry":"pretty_print_semaphore","repo":"congvvc/HyperSeg","repo_kind":"official","path":"hyperseg/model/mipha/utils.py","file_url":"https://github.com/congvvc/HyperSeg/blob/HEAD/hyperseg/model/mipha/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"37899f22fb191b37"}},{"code_sha256_prefix":"344dff4791fd1381","entry":"process_images","repo":"congvvc/HyperSeg","repo_kind":"official","path":"hyperseg/model/mipha/mm_utils.py","file_url":"https://github.com/congvvc/HyperSeg/blob/HEAD/hyperseg/model/mipha/mm_utils.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"344dff4791fd1381"}},{"code_sha256_prefix":"f9939a84b9a65279","entry":"violates_moderation","repo":"congvvc/HyperSeg","repo_kind":"official","path":"hyperseg/model/mipha/utils.py","file_url":"https://github.com/congvvc/HyperSeg/blob/HEAD/hyperseg/model/mipha/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f9939a84b9a65279"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}