{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/region-aware-pretraining-for-open-vocabulary","title":"Region-Aware Pretraining for Open-Vocabulary Object Detection with Vision Transformers","arxiv_id":"2305.07011","date":"2023-05-11","proceeding":"CVPR 2023 1","authors":["Dahun Kim","Anelia Angelova","Weicheng Kuo"],"abstract":"We present Region-aware Open-vocabulary Vision Transformers (RO-ViT) - a contrastive image-text pretraining recipe to bridge the gap between image-level pretraining and open-vocabulary object detection. At the pretraining phase, we propose to randomly crop and resize regions of positional embeddings instead of using the whole image positional embeddings. This better matches the use of positional embeddings at region-level in the detection finetuning phase. In addition, we replace the common softmax cross entropy loss in contrastive learning with focal loss to better learn the informative yet difficult examples. Finally, we leverage recent advances in novel object proposals to improve open-vocabulary detection finetuning. We evaluate our full model on the LVIS and COCO open-vocabulary detection benchmarks and zero-shot transfer. RO-ViT achieves a state-of-the-art 34.1 $AP_r$ on LVIS, surpassing the best existing approach by +7.8 points in addition to competitive zero-shot transfer detection. Surprisingly, RO-ViT improves the image-level representation as well and achieves the state of the art on 9 out of 12 metrics on COCO and Flickr image-text retrieval benchmarks, outperforming competitive approaches with larger models.","url_abs":"https://arxiv.org/abs/2305.07011v4","url_pdf":"https://arxiv.org/pdf/2305.07011v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"region-aware-pretraining-for-open-vocabulary","repo_url":"https://github.com/mcahny/rovit","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"jax","reach":{"status":"ok"}},{"paper_slug":"region-aware-pretraining-for-open-vocabulary","repo_url":"https://github.com/google-research/google-research/tree/master/fvlm/rovit","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"jax","reach":null}],"tasks":[{"task_slug":"contrastive-learning","task_name":"Contrastive Learning"},{"task_slug":"image-text-retrieval","task_name":"Image-text Retrieval"},{"task_slug":"object-detection","task_name":"Object Detection"},{"task_slug":"open-vocabulary-object-detection","task_name":"Open Vocabulary Object Detection"},{"task_slug":"open-vocabulary-object-detection","task_name":"Open-vocabulary object detection"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"text-retrieval","task_name":"Text Retrieval"},{"task_slug":"zero-shot-cross-modal-retrieval","task_name":"Zero-Shot Cross-Modal Retrieval"},{"task_slug":"object-detection-1","task_name":"object-detection"}],"methods":[{"method_slug":"contrastive-learning","method_name":"Contrastive Learning"},{"method_slug":"focal-loss","method_name":"Focal Loss"},{"method_slug":"softmax","method_name":"Softmax"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/open-vocabulary-object-detection-on-lvis-v1-0","task":"Open Vocabulary Object Detection","dataset":"LVIS v1.0","model":"RO-ViT","rank_in_archive_order":10,"of":28,"metrics":{"AP novel-LVIS base training":"32.1"},"uses_additional_data":false},{"leaderboard":"/sota/zero-shot-cross-modal-retrieval-on-coco-2014","task":"Zero-Shot Cross-Modal Retrieval","dataset":"COCO 2014","model":"RO-ViT","rank_in_archive_order":6,"of":18,"metrics":{"Image-to-text R@1":"68.9","Image-to-text R@10":"92.2","Image-to-text R@5":"87.8","Text-to-image R@1":"51.8","Text-to-image R@10":"83.0","Text-to-image R@5":"75.0"},"uses_additional_data":false},{"leaderboard":"/sota/zero-shot-cross-modal-retrieval-on-flickr30k","task":"Zero-Shot Cross-Modal Retrieval","dataset":"Flickr30k","model":"RO-ViT","rank_in_archive_order":6,"of":22,"metrics":{"Image-to-text R@1":"92.1","Image-to-text R@10":"99.7","Image-to-text R@5":"99.4","Text-to-image R@1":"80.7","Text-to-image R@10":"97.7","Text-to-image R@5":"96.1"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2305.07011","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.07011"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mcahny/rovit","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/google-research/google-research/tree/master/fvlm/rovit","reach":null}],"summary":{"ran":3,"ran_fixture":1,"ran_honours":1,"unverified":3},"by_repo_kind":{"official":{"samples":8,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"d8ad1a5da80c6c5a","entry":"Encoder","repo":"google-research/google-research","repo_kind":"official","path":"moe_mtl/modeling/vit.py","file_url":"https://github.com/google-research/google-research/blob/HEAD/moe_mtl/modeling/vit.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d8ad1a5da80c6c5a"}},{"code_sha256_prefix":"e2cdd60f8bf0cc83","entry":"Encoder1DBlock","repo":"google-research/google-research","repo_kind":"official","path":"moe_mtl/modeling/vit.py","file_url":"https://github.com/google-research/google-research/blob/HEAD/moe_mtl/modeling/vit.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e2cdd60f8bf0cc83"}},{"code_sha256_prefix":"6d80bcd9c94bd100","entry":"MlpBlock","repo":"google-research/google-research","repo_kind":"official","path":"moe_mtl/modeling/vit.py","file_url":"https://github.com/google-research/google-research/blob/HEAD/moe_mtl/modeling/vit.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6d80bcd9c94bd100"}},{"code_sha256_prefix":"a8f77cb87a6eb75b","entry":"get_posemb","repo":"google-research/google-research","repo_kind":"official","path":"moe_mtl/modeling/vit.py","file_url":"https://github.com/google-research/google-research/blob/HEAD/moe_mtl/modeling/vit.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a8f77cb87a6eb75b"}},{"code_sha256_prefix":"2d67a8a756fe7cf8","entry":"posemb_sincos_2d","repo":"google-research/google-research","repo_kind":"official","path":"moe_mtl/modeling/vit.py","file_url":"https://github.com/google-research/google-research/blob/HEAD/moe_mtl/modeling/vit.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2d67a8a756fe7cf8"}},{"code_sha256_prefix":"6aec67ee501d1501","entry":"Model","repo":"google-research/google-research","repo_kind":"official","path":"moe_mtl/modeling/vit.py","file_url":"https://github.com/google-research/google-research/blob/HEAD/moe_mtl/modeling/vit.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6aec67ee501d1501"}},{"code_sha256_prefix":"6c0a4f49af0bfb40","entry":"window_merge","repo":"google-research/google-research","repo_kind":"official","path":"moe_mtl/modeling/vit.py","file_url":"https://github.com/google-research/google-research/blob/HEAD/moe_mtl/modeling/vit.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6c0a4f49af0bfb40"}},{"code_sha256_prefix":"476400e99387d750","entry":"window_partition","repo":"google-research/google-research","repo_kind":"official","path":"moe_mtl/modeling/vit.py","file_url":"https://github.com/google-research/google-research/blob/HEAD/moe_mtl/modeling/vit.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"476400e99387d750"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}