{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/universal-instance-perception-as-object","title":"Universal Instance Perception as Object Discovery and Retrieval","arxiv_id":"2303.06674","date":"2023-03-12","proceeding":"CVPR 2023 1","authors":["Bin Yan","Yi Jiang","Jiannan Wu","Dong Wang","Ping Luo","Zehuan Yuan","Huchuan Lu"],"abstract":"All instance perception tasks aim at finding certain objects specified by some queries such as category names, language expressions, and target annotations, but this complete field has been split into multiple independent subtasks. In this work, we present a universal instance perception model of the next generation, termed UNINEXT. UNINEXT reformulates diverse instance perception tasks into a unified object discovery and retrieval paradigm and can flexibly perceive different types of objects by simply changing the input prompts. This unified formulation brings the following benefits: (1) enormous data from different tasks and label vocabularies can be exploited for jointly training general instance-level representations, which is especially beneficial for tasks lacking in training data. (2) the unified model is parameter-efficient and can save redundant computation when handling multiple tasks simultaneously. UNINEXT shows superior performance on 20 challenging benchmarks from 10 instance-level tasks including classical image-level tasks (object detection and instance segmentation), vision-and-language tasks (referring expression comprehension and segmentation), and six video-level object tracking tasks. Code is available at https://github.com/MasterBin-IIAU/UNINEXT.","url_abs":"https://arxiv.org/abs/2303.06674v2","url_pdf":"https://arxiv.org/pdf/2303.06674v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"universal-instance-perception-as-object","repo_url":"https://github.com/MasterBin-IIAU/UNINEXT","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"described-object-detection","task_name":"Described Object Detection"},{"task_slug":"generalized-referring-expression","task_name":"Generalized Referring Expression Comprehension"},{"task_slug":"instance-segmentation","task_name":"Instance Segmentation"},{"task_slug":"multi-object-tracking-and-segmentation","task_name":"Multi-Object Tracking and Segmentation"},{"task_slug":"multiple-object-tracking","task_name":"Multiple Object Tracking"},{"task_slug":"object","task_name":"Object"},{"task_slug":"object-detection","task_name":"Object Detection"},{"task_slug":"object-discovery","task_name":"Object Discovery"},{"task_slug":"object-tracking","task_name":"Object Tracking"},{"task_slug":"referring-expression","task_name":"Referring Expression"},{"task_slug":"referring-expression-comprehension","task_name":"Referring Expression Comprehension"},{"task_slug":"referring-expression-segmentation","task_name":"Referring Expression Segmentation"},{"task_slug":"referring-video-object-segmentation","task_name":"Referring Video Object Segmentation"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"},{"task_slug":"video-instance-segmentation","task_name":"Video Instance Segmentation"},{"task_slug":"visual-object-tracking","task_name":"Visual Object Tracking"},{"task_slug":"visual-tracking","task_name":"Visual Tracking"},{"task_slug":"zero-shot-segmentation","task_name":"Zero Shot Segmentation"},{"task_slug":"object-detection-1","task_name":"object-detection"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/described-object-detection-on-description","task":"Described Object Detection","dataset":"Description Detection Dataset","model":"UNINEXT-large","rank_in_archive_order":5,"of":8,"metrics":{"Intra-scenario ABS mAP":"15.9","Intra-scenario FULL mAP":"17.9","Intra-scenario PRES mAP":"18.6"},"uses_additional_data":false},{"leaderboard":"/sota/generalized-referring-expression","task":"Generalized Referring Expression Comprehension","dataset":"gRefCOCO","model":"UNINEXT","rank_in_archive_order":2,"of":5,"metrics":{"N-acc.":"50.6","Precision@(F1=1, IoU≥0.5)":"58.2"},"uses_additional_data":false},{"leaderboard":"/sota/instance-segmentation-on-coco","task":"Instance Segmentation","dataset":"COCO test-dev","model":"UNINEXT-H","rank_in_archive_order":17,"of":112,"metrics":{"AP50":"76.2","AP75":"56.7","APL":"67.5","APM":"55.9","APS":"33.3","mask AP":"51.8"},"uses_additional_data":true},{"leaderboard":"/sota/multi-object-tracking-and-segmentation-on-3","task":"Multi-Object Tracking and Segmentation","dataset":"BDD100K val","model":"UNINEXT-H","rank_in_archive_order":1,"of":8,"metrics":{"mMOTSA":"35.7"},"uses_additional_data":false},{"leaderboard":"/sota/multiple-object-tracking-on-bdd100k-val","task":"Multiple Object Tracking","dataset":"BDD100K val","model":"UNINEXT-H","rank_in_archive_order":2,"of":9,"metrics":{"AssocA":"-","TETA":"-","mIDF1":"56.7","mMOTA":"44.2"},"uses_additional_data":false},{"leaderboard":"/sota/object-detection-on-coco-minival","task":"Object Detection","dataset":"COCO minival","model":"UNINEXT-H","rank_in_archive_order":21,"of":220,"metrics":{"AP50":"77.5","AP75":"66.7","APL":"75.3","APM":"64.8","APS":"45.1","box AP":"60.6"},"uses_additional_data":true},{"leaderboard":"/sota/referring-expression-segmentation-on-davis","task":"Referring Expression Segmentation","dataset":"DAVIS 2017 (val)","model":"UNINEXT-H","rank_in_archive_order":1,"of":18,"metrics":{"J&F 1st frame":"72.5"},"uses_additional_data":true},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco-5","task":"Referring Expression Segmentation","dataset":"RefCOCO+ test B","model":"UNINEXT-H","rank_in_archive_order":8,"of":30,"metrics":{"Overall IoU":"66.22"},"uses_additional_data":true},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco-4","task":"Referring Expression Segmentation","dataset":"RefCOCO+ testA","model":"UNINEXT-H","rank_in_archive_order":9,"of":30,"metrics":{"Overall IoU":"76.42"},"uses_additional_data":true},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco-3","task":"Referring Expression Segmentation","dataset":"RefCOCO+ val","model":"UNINEXT-H","rank_in_archive_order":11,"of":33,"metrics":{"Overall IoU":"72.47"},"uses_additional_data":true},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco","task":"Referring Expression Segmentation","dataset":"RefCoCo val","model":"UNINEXT-H","rank_in_archive_order":7,"of":37,"metrics":{"Overall IoU":"82.19"},"uses_additional_data":true},{"leaderboard":"/sota/referring-expression-segmentation-on-refer-1","task":"Referring Expression Segmentation","dataset":"Refer-YouTube-VOS (2021 public validation)","model":"UNINEXT-H","rank_in_archive_order":4,"of":33,"metrics":{"F":"72.7","J":"67.6","J&F":"70.1"},"uses_additional_data":false},{"leaderboard":"/sota/video-instance-segmentation-on-ovis-1","task":"Video Instance Segmentation","dataset":"OVIS validation","model":"UNINEXT (ViT-H, Online)","rank_in_archive_order":7,"of":44,"metrics":{"AP50":"72.5","AP75":"52.2","mask AP":"49.0"},"uses_additional_data":true},{"leaderboard":"/sota/video-instance-segmentation-on-ovis-1","task":"Video Instance Segmentation","dataset":"OVIS validation","model":"UNINEXT (ResNet-50, Online)","rank_in_archive_order":26,"of":44,"metrics":{"AP50":"55.5","AP75":"35.6","mask AP":"34.0"},"uses_additional_data":true},{"leaderboard":"/sota/visual-object-tracking-on-lasot","task":"Visual Object Tracking","dataset":"LaSOT","model":"UNINEXT-L","rank_in_archive_order":19,"of":46,"metrics":{"AUC":"72.4","Normalized Precision":"80.7","Precision":"78.9"},"uses_additional_data":false},{"leaderboard":"/sota/visual-object-tracking-on-lasot","task":"Visual Object Tracking","dataset":"LaSOT","model":"UNINEXT-H","rank_in_archive_order":21,"of":46,"metrics":{"AUC":"72.2","Normalized Precision":"80.8","Precision":"79.4"},"uses_additional_data":false},{"leaderboard":"/sota/visual-object-tracking-on-lasot-ext","task":"Visual Object Tracking","dataset":"LaSOT-ext","model":"UNINEXT-H","rank_in_archive_order":5,"of":18,"metrics":{"AUC":"56.2","Normalized Precision":"63.8","Precision":"63.8"},"uses_additional_data":false},{"leaderboard":"/sota/visual-object-tracking-on-trackingnet","task":"Visual Object Tracking","dataset":"TrackingNet","model":"UNINEXT-H","rank_in_archive_order":13,"of":40,"metrics":{"Accuracy":"85.4","Normalized Precision":"89.0","Precision":"86.4"},"uses_additional_data":false},{"leaderboard":"/sota/visual-tracking-on-tnl2k","task":"Visual Tracking","dataset":"TNL2K","model":"UNINEXT-H","rank_in_archive_order":2,"of":6,"metrics":{"AUC":"59.3","precision":"62.8"},"uses_additional_data":false},{"leaderboard":"/sota/zero-shot-segmentation-on-segmentation-in-the","task":"Zero Shot Segmentation","dataset":"Segmentation in the Wild","model":"UNINEXT","rank_in_archive_order":3,"of":12,"metrics":{"Mean AP":"42.1"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2303.06674","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2303.06674"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/MasterBin-IIAU/UNINEXT","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":1,"ran_honours":1,"ran":1,"unverified":1},"by_repo_kind":{"official":{"samples":4,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"29dbbaafa4be18e1","entry":"aligned_bilinear","repo":"MasterBin-IIAU/UNINEXT","repo_kind":"official","path":"projects/UNINEXT/uninext/models/ddetrs.py","file_url":"https://github.com/MasterBin-IIAU/UNINEXT/blob/HEAD/projects/UNINEXT/uninext/models/ddetrs.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"29dbbaafa4be18e1"}},{"code_sha256_prefix":"ed1a308adb65f645","entry":"compute_locations","repo":"MasterBin-IIAU/UNINEXT","repo_kind":"official","path":"projects/UNINEXT/uninext/models/ddetrs.py","file_url":"https://github.com/MasterBin-IIAU/UNINEXT/blob/HEAD/projects/UNINEXT/uninext/models/ddetrs.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ed1a308adb65f645"}},{"code_sha256_prefix":"46f34dc9f9a94b67","entry":"parse_dynamic_params","repo":"MasterBin-IIAU/UNINEXT","repo_kind":"official","path":"projects/UNINEXT/uninext/models/ddetrs.py","file_url":"https://github.com/MasterBin-IIAU/UNINEXT/blob/HEAD/projects/UNINEXT/uninext/models/ddetrs.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"46f34dc9f9a94b67"}},{"code_sha256_prefix":"8edb076e926c9e81","entry":"make_full_size","repo":"MasterBin-IIAU/UNINEXT","repo_kind":"official","path":"uninext_vots.py","file_url":"https://github.com/MasterBin-IIAU/UNINEXT/blob/HEAD/uninext_vots.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8edb076e926c9e81"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}