{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/learning-customized-visual-models-with","title":"Learning Customized Visual Models with Retrieval-Augmented Knowledge","arxiv_id":"2301.07094","date":"2023-01-17","proceeding":"CVPR 2023 1","authors":["Haotian Liu","Kilho Son","Jianwei Yang","Ce Liu","Jianfeng Gao","Yong Jae Lee","Chunyuan Li"],"abstract":"Image-text contrastive learning models such as CLIP have demonstrated strong task transfer ability. The high generality and usability of these visual models is achieved via a web-scale data collection process to ensure broad concept coverage, followed by expensive pre-training to feed all the knowledge into model weights. Alternatively, we propose REACT, REtrieval-Augmented CusTomization, a framework to acquire the relevant web knowledge to build customized visual models for target domains. We retrieve the most relevant image-text pairs (~3% of CLIP pre-training data) from the web-scale database as external knowledge, and propose to customize the model by only training new modualized blocks while freezing all the original weights. The effectiveness of REACT is demonstrated via extensive experiments on classification, retrieval, detection and segmentation tasks, including zero, few, and full-shot settings. Particularly, on the zero-shot classification task, compared with CLIP, it achieves up to 5.4% improvement on ImageNet and 3.7% on the ELEVATER benchmark (20 datasets).","url_abs":"https://arxiv.org/abs/2301.07094v1","url_pdf":"https://arxiv.org/pdf/2301.07094v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"learning-customized-visual-models-with","repo_url":"https://github.com/microsoft/react","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"contrastive-learning","task_name":"Contrastive Learning"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"semi-supervised-image-classification","task_name":"Semi-Supervised Image Classification"},{"task_slug":"zero-shot-learning","task_name":"Zero-Shot Learning"},{"task_slug":"zero-shot-transfer-image-classification","task_name":"Zero-Shot Transfer Image Classification"},{"task_slug":null,"task_name":"zero-shot-classification"}],"methods":[{"method_slug":"clip","method_name":"CLIP"},{"method_slug":"contrastive-learning","method_name":"Contrastive Learning"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/semi-supervised-image-classification-on-1","task":"Semi-Supervised Image Classification","dataset":"ImageNet - 1% labeled data","model":"REACT (ViT-Large)","rank_in_archive_order":2,"of":65,"metrics":{"Top 1 Accuracy":"81.6%"},"uses_additional_data":true},{"leaderboard":"/sota/semi-supervised-image-classification-on-2","task":"Semi-Supervised Image Classification","dataset":"ImageNet - 10% labeled data","model":"REACT (ViT-Large)","rank_in_archive_order":3,"of":75,"metrics":{"Top 1 Accuracy":"85.1%"},"uses_additional_data":true},{"leaderboard":"/sota/zero-shot-transfer-image-classification-on-1","task":"Zero-Shot Transfer Image Classification","dataset":"ImageNet","model":"REACT","rank_in_archive_order":15,"of":23,"metrics":{"Accuracy (Private)":"78.5"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2301.07094","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2301.07094"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/microsoft/react","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":4,"ran_violates":1,"ran":1,"unverified":11},"by_repo_kind":{"official":{"samples":17,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"98f385d847636a3e","entry":"basic_clean","repo":"microsoft/react","repo_kind":"official","path":"react_customization/src/open_clip/tokenizer.py","file_url":"https://github.com/microsoft/react/blob/HEAD/react_customization/src/open_clip/tokenizer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"98f385d847636a3e"}},{"code_sha256_prefix":"f19962ebb134b3d7","entry":"convert_to_custom_text_state_dict","repo":"microsoft/react","repo_kind":"official","path":"react_customization/src/open_clip/model.py","file_url":"https://github.com/microsoft/react/blob/HEAD/react_customization/src/open_clip/model.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f19962ebb134b3d7"}},{"code_sha256_prefix":"dcd422d66b0581d8","entry":"get_cast_dtype","repo":"microsoft/react","repo_kind":"official","path":"react_customization/src/open_clip/model.py","file_url":"https://github.com/microsoft/react/blob/HEAD/react_customization/src/open_clip/model.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dcd422d66b0581d8"}},{"code_sha256_prefix":"d919ae32e5e4e616","entry":"get_pairs","repo":"microsoft/react","repo_kind":"official","path":"react_customization/src/open_clip/tokenizer.py","file_url":"https://github.com/microsoft/react/blob/HEAD/react_customization/src/open_clip/tokenizer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d919ae32e5e4e616"}},{"code_sha256_prefix":"4b3a4d17e1df3140","entry":"load_state_dict","repo":"microsoft/react","repo_kind":"official","path":"react_customization/src/open_clip/factory.py","file_url":"https://github.com/microsoft/react/blob/HEAD/react_customization/src/open_clip/factory.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4b3a4d17e1df3140"}},{"code_sha256_prefix":"9542161e9640b858","entry":"whitespace_clean","repo":"microsoft/react","repo_kind":"official","path":"react_customization/src/open_clip/tokenizer.py","file_url":"https://github.com/microsoft/react/blob/HEAD/react_customization/src/open_clip/tokenizer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9542161e9640b858"}},{"code_sha256_prefix":"26a799d4c5d477a1","entry":"build_model_from_openai_state_dict","repo":"microsoft/react","repo_kind":"official","path":"react_customization/src/open_clip/model.py","file_url":"https://github.com/microsoft/react/blob/HEAD/react_customization/src/open_clip/model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"26a799d4c5d477a1"}},{"code_sha256_prefix":"ddcbd45e940484ee","entry":"gather_features","repo":"microsoft/react","repo_kind":"official","path":"react_customization/src/open_clip/loss.py","file_url":"https://github.com/microsoft/react/blob/HEAD/react_customization/src/open_clip/loss.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ddcbd45e940484ee"}},{"code_sha256_prefix":"f41a0f1396c97cb4","entry":"generate_readme","repo":"microsoft/react","repo_kind":"official","path":"react_customization/src/open_clip/push_to_hf_hub.py","file_url":"https://github.com/microsoft/react/blob/HEAD/react_customization/src/open_clip/push_to_hf_hub.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f41a0f1396c97cb4"}},{"code_sha256_prefix":"5c7cf820e99443ff","entry":"get_model_config","repo":"microsoft/react","repo_kind":"official","path":"react_customization/src/open_clip/factory.py","file_url":"https://github.com/microsoft/react/blob/HEAD/react_customization/src/open_clip/factory.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5c7cf820e99443ff"}},{"code_sha256_prefix":"f34d8f34635af394","entry":"get_tokenizer","repo":"microsoft/react","repo_kind":"official","path":"react_customization/src/open_clip/factory.py","file_url":"https://github.com/microsoft/react/blob/HEAD/react_customization/src/open_clip/factory.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f34d8f34635af394"}},{"code_sha256_prefix":"0131858929a8ca1e","entry":"list_pretrained","repo":"microsoft/react","repo_kind":"official","path":"react_customization/src/open_clip/pretrained.py","file_url":"https://github.com/microsoft/react/blob/HEAD/react_customization/src/open_clip/pretrained.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0131858929a8ca1e"}},{"code_sha256_prefix":"219f782c556ae686","entry":"list_pretrained_models_by_tag","repo":"microsoft/react","repo_kind":"official","path":"react_customization/src/open_clip/pretrained.py","file_url":"https://github.com/microsoft/react/blob/HEAD/react_customization/src/open_clip/pretrained.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"219f782c556ae686"}},{"code_sha256_prefix":"3e68e3ec56ca86ee","entry":"list_pretrained_tags_by_model","repo":"microsoft/react","repo_kind":"official","path":"react_customization/src/open_clip/pretrained.py","file_url":"https://github.com/microsoft/react/blob/HEAD/react_customization/src/open_clip/pretrained.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3e68e3ec56ca86ee"}},{"code_sha256_prefix":"a1693b09a14a516f","entry":"load_openai_model","repo":"microsoft/react","repo_kind":"official","path":"react_customization/src/open_clip/openai.py","file_url":"https://github.com/microsoft/react/blob/HEAD/react_customization/src/open_clip/openai.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a1693b09a14a516f"}},{"code_sha256_prefix":"fb651d0a97fd4d3f","entry":"prepare_inputs_for_generation","repo":"microsoft/react","repo_kind":"official","path":"react_customization/src/open_clip/coca_model.py","file_url":"https://github.com/microsoft/react/blob/HEAD/react_customization/src/open_clip/coca_model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fb651d0a97fd4d3f"}},{"code_sha256_prefix":"2a377da4a76a2d44","entry":"register_pooler","repo":"microsoft/react","repo_kind":"official","path":"react_customization/src/open_clip/hf_model.py","file_url":"https://github.com/microsoft/react/blob/HEAD/react_customization/src/open_clip/hf_model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2a377da4a76a2d44"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}