{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/general-object-foundation-model-for-images","title":"General Object Foundation Model for Images and Videos at Scale","arxiv_id":"2312.09158","date":"2023-12-14","proceeding":"CVPR 2024 1","authors":["Junfeng Wu","Yi Jiang","Qihao Liu","Zehuan Yuan","Xiang Bai","Song Bai"],"abstract":"We present GLEE in this work, an object-level foundation model for locating and identifying objects in images and videos. Through a unified framework, GLEE accomplishes detection, segmentation, tracking, grounding, and identification of arbitrary objects in the open world scenario for various object perception tasks. Adopting a cohesive learning strategy, GLEE acquires knowledge from diverse data sources with varying supervision levels to formulate general object representations, excelling in zero-shot transfer to new data and tasks. Specifically, we employ an image encoder, text encoder, and visual prompter to handle multi-modal inputs, enabling to simultaneously solve various object-centric downstream tasks while maintaining state-of-the-art performance. Demonstrated through extensive training on over five million images from diverse benchmarks, GLEE exhibits remarkable versatility and improved generalization performance, efficiently tackling downstream tasks without the need for task-specific adaptation. By integrating large volumes of automatically labeled data, we further enhance its zero-shot generalization capabilities. Additionally, GLEE is capable of being integrated into Large Language Models, serving as a foundational model to provide universal object-level information for multi-modal tasks. We hope that the versatility and universality of our method will mark a significant step in the development of efficient visual foundation models for AGI systems. The model and code will be released at https://glee-vision.github.io .","url_abs":"https://arxiv.org/abs/2312.09158v1","url_pdf":"https://arxiv.org/pdf/2312.09158v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"general-object-foundation-model-for-images","repo_url":"https://github.com/FoundationVision/GLEE","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"instance-segmentation","task_name":"Instance Segmentation"},{"task_slug":"long-tail-video-object-segmentation","task_name":"Long-tail Video Object Segmentation"},{"task_slug":"multi-object-tracking","task_name":"Multi-Object Tracking"},{"task_slug":"object","task_name":"Object"},{"task_slug":"object-detection","task_name":"Object Detection"},{"task_slug":"open-world-instance-segmentation","task_name":"Open-World Instance Segmentation"},{"task_slug":"referring-expression-comprehension","task_name":"Referring Expression Comprehension"},{"task_slug":"referring-expression-segmentation","task_name":"Referring Expression Segmentation"},{"task_slug":"referring-video-object-segmentation","task_name":"Referring Video Object Segmentation"},{"task_slug":"video-instance-segmentation","task_name":"Video Instance Segmentation"},{"task_slug":"video-object-segmentation","task_name":"Video Object Segmentation"},{"task_slug":"zero-shot-generalization","task_name":"Zero-shot Generalization"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/instance-segmentation-on-coco-minival","task":"Instance Segmentation","dataset":"COCO minival","model":"GLEE-Pro","rank_in_archive_order":8,"of":93,"metrics":{"mask AP":"54.2"},"uses_additional_data":true},{"leaderboard":"/sota/instance-segmentation-on-coco-minival","task":"Instance Segmentation","dataset":"COCO minival","model":"GLEE-Plus","rank_in_archive_order":11,"of":93,"metrics":{"mask AP":"53.0"},"uses_additional_data":true},{"leaderboard":"/sota/instance-segmentation-on-coco-minival","task":"Instance Segmentation","dataset":"COCO minival","model":"GLEE-Lite","rank_in_archive_order":37,"of":93,"metrics":{"mask AP":"48.4"},"uses_additional_data":true},{"leaderboard":"/sota/instance-segmentation-on-coco","task":"Instance Segmentation","dataset":"COCO test-dev","model":"GLEE-Pro","rank_in_archive_order":9,"of":112,"metrics":{"mask AP":"54.5"},"uses_additional_data":true},{"leaderboard":"/sota/instance-segmentation-on-coco","task":"Instance Segmentation","dataset":"COCO test-dev","model":"GLEE-Plus","rank_in_archive_order":11,"of":112,"metrics":{"mask AP":"53.3"},"uses_additional_data":true},{"leaderboard":"/sota/instance-segmentation-on-coco","task":"Instance Segmentation","dataset":"COCO test-dev","model":"GLEE-Lite","rank_in_archive_order":31,"of":112,"metrics":{"mask AP":"48.3"},"uses_additional_data":true},{"leaderboard":"/sota/instance-segmentation-on-lvis-v1-0-val","task":"Instance Segmentation","dataset":"LVIS v1.0 val","model":"GLEE-Pro","rank_in_archive_order":4,"of":25,"metrics":{"mask AP":"49.9"},"uses_additional_data":true},{"leaderboard":"/sota/long-tail-video-object-segmentation-on-burst","task":"Long-tail Video Object Segmentation","dataset":"BURST","model":"GLEE-Lite","rank_in_archive_order":1,"of":1,"metrics":{"HOTA (all)":"22.6","HOTA (com)":"36.4","HOTA (unc)":"19.1","mAP (all)":"12.6","mAP (com)":"18.9","mAP (unc)":"11.0"},"uses_additional_data":true},{"leaderboard":"/sota/long-tail-video-object-segmentation-on-burst-1","task":"Long-tail Video Object Segmentation","dataset":"BURST-val","model":"GLEE-Pro","rank_in_archive_order":1,"of":5,"metrics":{"HOTA (all)":"31.2","HOTA (com)":"48.7","HOTA (unc)":"26.9","mAP (all)":"19.2","mAP (com)":"24.8","mAP (unc)":"17.7"},"uses_additional_data":true},{"leaderboard":"/sota/long-tail-video-object-segmentation-on-burst-1","task":"Long-tail Video Object Segmentation","dataset":"BURST-val","model":"GLEE-Plus","rank_in_archive_order":2,"of":5,"metrics":{"HOTA (all)":"26.9","HOTA (com)":"38.8","HOTA (unc)":"23.9","mAP (all)":"17.2","mAP (com)":"23.7","mAP (unc)":"15.5"},"uses_additional_data":true},{"leaderboard":"/sota/long-tail-video-object-segmentation-on-burst-1","task":"Long-tail Video Object Segmentation","dataset":"BURST-val","model":"GLEE-Lite","rank_in_archive_order":3,"of":5,"metrics":{"HOTA (all)":"22.6","HOTA (com)":"36.4","HOTA (unc)":"19.1","mAP (all)":"12.6","mAP (com)":"18.9","mAP (unc)":"11.0"},"uses_additional_data":true},{"leaderboard":"/sota/multi-object-tracking-on-tao","task":"Multi-Object Tracking","dataset":"TAO","model":"GLEE-Pro","rank_in_archive_order":2,"of":9,"metrics":{"AssocA":"46.2","ClsA":"29.1","LocA":"66.2","TETA":"47.2"},"uses_additional_data":true},{"leaderboard":"/sota/multi-object-tracking-on-tao","task":"Multi-Object Tracking","dataset":"TAO","model":"GLEE-Plus","rank_in_archive_order":3,"of":9,"metrics":{"AssocA":"40.9","ClsA":"30.8","LocA":"52.9","TETA":"41.5"},"uses_additional_data":true},{"leaderboard":"/sota/multi-object-tracking-on-tao","task":"Multi-Object Tracking","dataset":"TAO","model":"GLEE-Lite","rank_in_archive_order":4,"of":9,"metrics":{"AssocA":"39.9","ClsA":"24.1","LocA":"56.3","TETA":"40.1"},"uses_additional_data":true},{"leaderboard":"/sota/object-detection-on-coco-minival","task":"Object Detection","dataset":"COCO minival","model":"GLEE-Pro","rank_in_archive_order":17,"of":220,"metrics":{"box AP":"62.0"},"uses_additional_data":true},{"leaderboard":"/sota/object-detection-on-coco-minival","task":"Object Detection","dataset":"COCO minival","model":"GLEE-Plus","rank_in_archive_order":24,"of":220,"metrics":{"box AP":"60.4"},"uses_additional_data":true},{"leaderboard":"/sota/object-detection-on-coco-minival","task":"Object Detection","dataset":"COCO minival","model":"GLEE-Lite","rank_in_archive_order":53,"of":220,"metrics":{"box AP":"55.0"},"uses_additional_data":true},{"leaderboard":"/sota/object-detection-on-coco","task":"Object Detection","dataset":"COCO test-dev","model":"GLEE-Pro","rank_in_archive_order":22,"of":225,"metrics":{"box mAP":"62.3"},"uses_additional_data":false},{"leaderboard":"/sota/object-detection-on-coco","task":"Object Detection","dataset":"COCO test-dev","model":"GLEE-Plus","rank_in_archive_order":27,"of":225,"metrics":{"box mAP":"60.6"},"uses_additional_data":false},{"leaderboard":"/sota/object-detection-on-coco","task":"Object Detection","dataset":"COCO test-dev","model":"GLEE-Lite","rank_in_archive_order":54,"of":225,"metrics":{"box mAP":"54.7"},"uses_additional_data":false},{"leaderboard":"/sota/object-detection-on-lvis-v1-0-val","task":"Object Detection","dataset":"LVIS v1.0 val","model":"GLEE-Pro","rank_in_archive_order":6,"of":15,"metrics":{"box AP":"55.7"},"uses_additional_data":true},{"leaderboard":"/sota/open-world-instance-segmentation-on-uvo","task":"Open-World Instance Segmentation","dataset":"UVO","model":"GLEE-Pro","rank_in_archive_order":1,"of":2,"metrics":{"ARmask":"72.6"},"uses_additional_data":true},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco-6","task":"Referring Expression Segmentation","dataset":"RefCOCO","model":"GLEE-Pro","rank_in_archive_order":2,"of":4,"metrics":{"IoU":"80.0"},"uses_additional_data":true},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco-3","task":"Referring Expression Segmentation","dataset":"RefCOCO+ val","model":"GLEE-Pro","rank_in_archive_order":15,"of":33,"metrics":{"Overall IoU":"69.6"},"uses_additional_data":true},{"leaderboard":"/sota/referring-expression-segmentation-on-refcocog","task":"Referring Expression Segmentation","dataset":"RefCOCOg-val","model":"GLEE-Pro","rank_in_archive_order":10,"of":23,"metrics":{"Overall IoU":"72.9"},"uses_additional_data":true},{"leaderboard":"/sota/referring-expression-segmentation-on-refcoco","task":"Referring Expression Segmentation","dataset":"RefCoCo val","model":"GLEE-Pro","rank_in_archive_order":11,"of":37,"metrics":{"Overall IoU":"80.0"},"uses_additional_data":true},{"leaderboard":"/sota/referring-expression-segmentation-on-refer-1","task":"Referring Expression Segmentation","dataset":"Refer-YouTube-VOS (2021 public validation)","model":"GLEE-Pro","rank_in_archive_order":3,"of":33,"metrics":{"F":"72.9","J":"68.2","J&F":"70.6"},"uses_additional_data":true},{"leaderboard":"/sota/referring-video-object-segmentation-on-refer","task":"Referring Video Object Segmentation","dataset":"Refer-YouTube-VOS","model":"GLEE-Pro","rank_in_archive_order":2,"of":18,"metrics":{"F":"72.9","J":"68.2","J&F":"70.6"},"uses_additional_data":true},{"leaderboard":"/sota/referring-video-object-segmentation-on-refer","task":"Referring Video Object Segmentation","dataset":"Refer-YouTube-VOS","model":"GLEE-Plus","rank_in_archive_order":4,"of":18,"metrics":{"F":"69.7","J":"65.6","J&F":"67.7"},"uses_additional_data":true},{"leaderboard":"/sota/video-instance-segmentation-on-ovis-1","task":"Video Instance Segmentation","dataset":"OVIS validation","model":"GLEE-Pro","rank_in_archive_order":4,"of":44,"metrics":{"AP75":"55.5","mask AP":"50.4"},"uses_additional_data":true}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2312.09158","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.09158"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/FoundationVision/GLEE","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":6,"ran_draft_wrong":2,"unverified":5},"by_repo_kind":{"official":{"samples":13,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"95771c1a2d8346b7","entry":"agg_lang_feat","repo":"FoundationVision/GLEE","repo_kind":"official","path":"projects/GLEE/glee/models/glee_model.py","file_url":"https://github.com/FoundationVision/GLEE/blob/HEAD/projects/GLEE/glee/models/glee_model.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"95771c1a2d8346b7"}},{"code_sha256_prefix":"29dbbaafa4be18e1","entry":"aligned_bilinear","repo":"FoundationVision/GLEE","repo_kind":"official","path":"projects/GLEE/glee/models/vos_utils.py","file_url":"https://github.com/FoundationVision/GLEE/blob/HEAD/projects/GLEE/glee/models/vos_utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"29dbbaafa4be18e1"}},{"code_sha256_prefix":"42ab1750d0229462","entry":"generate_regular_grid_point_coords","repo":"FoundationVision/GLEE","repo_kind":"official","path":"projects/GLEE/glee/modules/point_features.py","file_url":"https://github.com/FoundationVision/GLEE/blob/HEAD/projects/GLEE/glee/modules/point_features.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"42ab1750d0229462"}},{"code_sha256_prefix":"670bf62deb1d06af","entry":"load_opt_from_config_files","repo":"FoundationVision/GLEE","repo_kind":"official","path":"projects/GLEE/train_net.py","file_url":"https://github.com/FoundationVision/GLEE/blob/HEAD/projects/GLEE/train_net.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"670bf62deb1d06af"}},{"code_sha256_prefix":"3e759ab7f3a7de8d","entry":"masks_to_boxes","repo":"FoundationVision/GLEE","repo_kind":"official","path":"projects/GLEE/glee/models/vos_utils.py","file_url":"https://github.com/FoundationVision/GLEE/blob/HEAD/projects/GLEE/glee/models/vos_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3e759ab7f3a7de8d"}},{"code_sha256_prefix":"39e4af93b70bcddf","entry":"point_sample","repo":"FoundationVision/GLEE","repo_kind":"official","path":"projects/GLEE/glee/modules/point_features.py","file_url":"https://github.com/FoundationVision/GLEE/blob/HEAD/projects/GLEE/glee/modules/point_features.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"39e4af93b70bcddf"}},{"code_sha256_prefix":"130323eef819c472","entry":"rand_sample","repo":"FoundationVision/GLEE","repo_kind":"official","path":"projects/GLEE/glee/models/glee_model.py","file_url":"https://github.com/FoundationVision/GLEE/blob/HEAD/projects/GLEE/glee/models/glee_model.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"130323eef819c472"}},{"code_sha256_prefix":"5c0711aada67957e","entry":"sigmoid_focal_loss","repo":"FoundationVision/GLEE","repo_kind":"official","path":"projects/GLEE/glee/models/criterion.py","file_url":"https://github.com/FoundationVision/GLEE/blob/HEAD/projects/GLEE/glee/models/criterion.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5c0711aada67957e"}},{"code_sha256_prefix":"bc2cb481a75c370d","entry":"batch_dice_loss","repo":"FoundationVision/GLEE","repo_kind":"official","path":"projects/GLEE/glee/models/matcher.py","file_url":"https://github.com/FoundationVision/GLEE/blob/HEAD/projects/GLEE/glee/models/matcher.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bc2cb481a75c370d"}},{"code_sha256_prefix":"1edd24985036b0bf","entry":"batch_sigmoid_ce_loss","repo":"FoundationVision/GLEE","repo_kind":"official","path":"projects/GLEE/glee/models/matcher.py","file_url":"https://github.com/FoundationVision/GLEE/blob/HEAD/projects/GLEE/glee/models/matcher.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1edd24985036b0bf"}},{"code_sha256_prefix":"89f75e54ff128be0","entry":"dice_loss","repo":"FoundationVision/GLEE","repo_kind":"official","path":"projects/GLEE/glee/models/criterion.py","file_url":"https://github.com/FoundationVision/GLEE/blob/HEAD/projects/GLEE/glee/models/criterion.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"89f75e54ff128be0"}},{"code_sha256_prefix":"22a94fc69aed9aca","entry":"multi_head_attention_forward","repo":"FoundationVision/GLEE","repo_kind":"official","path":"projects/GLEE/glee/modules/attention.py","file_url":"https://github.com/FoundationVision/GLEE/blob/HEAD/projects/GLEE/glee/modules/attention.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"22a94fc69aed9aca"}},{"code_sha256_prefix":"d0c61e8dba511aa3","entry":"sigmoid_ce_loss","repo":"FoundationVision/GLEE","repo_kind":"official","path":"projects/GLEE/glee/models/criterion.py","file_url":"https://github.com/FoundationVision/GLEE/blob/HEAD/projects/GLEE/glee/models/criterion.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d0c61e8dba511aa3"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}