{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/unsupervised-open-vocabulary-object","title":"Unsupervised Open-Vocabulary Object Localization in Videos","arxiv_id":"2309.09858","date":"2023-09-18","proceeding":"ICCV 2023 1","authors":["Ke Fan","Zechen Bai","Tianjun Xiao","Dominik Zietlow","Max Horn","Zixu Zhao","Carl-Johann Simon-Gabriel","Mike Zheng Shou","Francesco Locatello","Bernt Schiele","Thomas Brox","Zheng Zhang","Yanwei Fu","Tong He"],"abstract":"In this paper, we show that recent advances in video representation learning and pre-trained vision-language models allow for substantial improvements in self-supervised video object localization. We propose a method that first localizes objects in videos via an object-centric approach with slot attention and then assigns text to the obtained slots. The latter is achieved by an unsupervised way to read localized semantic information from the pre-trained CLIP model. The resulting video object localization is entirely unsupervised apart from the implicit annotation contained in CLIP, and it is effectively the first unsupervised approach that yields good results on regular video benchmarks.","url_abs":"https://arxiv.org/abs/2309.09858v2","url_pdf":"https://arxiv.org/pdf/2309.09858v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"unsupervised-open-vocabulary-object","repo_url":"https://github.com/amazon-science/object-centric-vol","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"object","task_name":"Object"},{"task_slug":"object-localization","task_name":"Object Localization"},{"task_slug":"representation-learning","task_name":"Representation Learning"}],"methods":[{"method_slug":"clip","method_name":"CLIP"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2309.09858","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2309.09858"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/amazon-science/object-centric-vol","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":11,"ran_fixture":1,"ran_honours":1,"unverified":2},"by_repo_kind":{"official":{"samples":15,"ran":13,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"4183dd075de22fc1","entry":"Generate_Graph_ST","repo":"amazon-science/object-centric-vol","repo_kind":"official","path":"post_processing.py","file_url":"https://github.com/amazon-science/object-centric-vol/blob/HEAD/post_processing.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4183dd075de22fc1"}},{"code_sha256_prefix":"131a82fd65128218","entry":"accuracy","repo":"amazon-science/object-centric-vol","repo_kind":"official","path":"train_patch_clip.py","file_url":"https://github.com/amazon-science/object-centric-vol/blob/HEAD/train_patch_clip.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"131a82fd65128218"}},{"code_sha256_prefix":"8f59f1b08fa79918","entry":"adjusted_rand_index","repo":"amazon-science/object-centric-vol","repo_kind":"official","path":"evaluation/grouping_metrics.py","file_url":"https://github.com/amazon-science/object-centric-vol/blob/HEAD/evaluation/grouping_metrics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8f59f1b08fa79918"}},{"code_sha256_prefix":"1d8a6c99853d9058","entry":"build_mlp","repo":"amazon-science/object-centric-vol","repo_kind":"official","path":"models/model_grouping.py","file_url":"https://github.com/amazon-science/object-centric-vol/blob/HEAD/models/model_grouping.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1d8a6c99853d9058"}},{"code_sha256_prefix":"756383bd57f5bc69","entry":"build_two_layer_mlp","repo":"amazon-science/object-centric-vol","repo_kind":"official","path":"models/model_grouping.py","file_url":"https://github.com/amazon-science/object-centric-vol/blob/HEAD/models/model_grouping.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"756383bd57f5bc69"}},{"code_sha256_prefix":"94db904018002114","entry":"get_activation_fn","repo":"amazon-science/object-centric-vol","repo_kind":"official","path":"models/model_grouping.py","file_url":"https://github.com/amazon-science/object-centric-vol/blob/HEAD/models/model_grouping.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"94db904018002114"}},{"code_sha256_prefix":"da651e3979a18f84","entry":"get_sinusoid_encoding_table","repo":"amazon-science/object-centric-vol","repo_kind":"official","path":"models/videomae.py","file_url":"https://github.com/amazon-science/object-centric-vol/blob/HEAD/models/videomae.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"da651e3979a18f84"}},{"code_sha256_prefix":"deb6d8812cc4725f","entry":"get_sinusoid_encoding_table_1d","repo":"amazon-science/object-centric-vol","repo_kind":"official","path":"models/videomae.py","file_url":"https://github.com/amazon-science/object-centric-vol/blob/HEAD/models/videomae.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"deb6d8812cc4725f"}},{"code_sha256_prefix":"f6be533bf96a449b","entry":"get_sinusoid_encoding_table_2d","repo":"amazon-science/object-centric-vol","repo_kind":"official","path":"models/videomae.py","file_url":"https://github.com/amazon-science/object-centric-vol/blob/HEAD/models/videomae.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f6be533bf96a449b"}},{"code_sha256_prefix":"4dc315c44ff26c83","entry":"merge_mask","repo":"amazon-science/object-centric-vol","repo_kind":"official","path":"post_processing.py","file_url":"https://github.com/amazon-science/object-centric-vol/blob/HEAD/post_processing.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4dc315c44ff26c83"}},{"code_sha256_prefix":"ffba20ee5aa7b4f8","entry":"resize_patches_to_image","repo":"amazon-science/object-centric-vol","repo_kind":"official","path":"evaluation/grouping_metrics.py","file_url":"https://github.com/amazon-science/object-centric-vol/blob/HEAD/evaluation/grouping_metrics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ffba20ee5aa7b4f8"}},{"code_sha256_prefix":"c7f9a5b82487aac3","entry":"tensor_to_one_hot","repo":"amazon-science/object-centric-vol","repo_kind":"official","path":"evaluation/grouping_metrics.py","file_url":"https://github.com/amazon-science/object-centric-vol/blob/HEAD/evaluation/grouping_metrics.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c7f9a5b82487aac3"}},{"code_sha256_prefix":"ddb29c73d68308d5","entry":"validate","repo":"amazon-science/object-centric-vol","repo_kind":"official","path":"train_patch_clip.py","file_url":"https://github.com/amazon-science/object-centric-vol/blob/HEAD/train_patch_clip.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ddb29c73d68308d5"}},{"code_sha256_prefix":"183ad4ca38a22a1b","entry":"get_model","repo":"amazon-science/object-centric-vol","repo_kind":"official","path":"run_mae_pretraining_single_frame.py","file_url":"https://github.com/amazon-science/object-centric-vol/blob/HEAD/run_mae_pretraining_single_frame.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"183ad4ca38a22a1b"}},{"code_sha256_prefix":"c4a39146549ee56f","entry":"merge_criterion","repo":"amazon-science/object-centric-vol","repo_kind":"official","path":"post_processing.py","file_url":"https://github.com/amazon-science/object-centric-vol/blob/HEAD/post_processing.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c4a39146549ee56f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}