{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/rwkv-clip-a-robust-vision-language","title":"RWKV-CLIP: A Robust Vision-Language Representation Learner","arxiv_id":"2406.06973","date":"2024-06-11","proceeding":null,"authors":["Tiancheng Gu","Kaicheng Yang","Xiang An","Ziyong Feng","Dongnan Liu","Weidong Cai","Jiankang Deng"],"abstract":"Contrastive Language-Image Pre-training (CLIP) has significantly improved performance in various vision-language tasks by expanding the dataset with image-text pairs obtained from websites. This paper further explores CLIP from the perspectives of data and model architecture. To address the prevalence of noisy data and enhance the quality of large-scale image-text data crawled from the internet, we introduce a diverse description generation framework that can leverage Large Language Models (LLMs) to synthesize and refine content from web-based texts, synthetic captions, and detection tags. Furthermore, we propose RWKV-CLIP, the first RWKV-driven vision-language representation learning model that combines the effective parallel training of transformers with the efficient inference of RNNs. Comprehensive experiments across various model scales and pre-training datasets demonstrate that RWKV-CLIP is a robust and efficient vision-language representation learner, it achieves state-of-the-art performance in several downstream tasks, including linear probe, zero-shot classification, and zero-shot image-text retrieval. To facilitate future research, the code and pre-trained models are released at https://github.com/deepglint/RWKV-CLIP","url_abs":"https://arxiv.org/abs/2406.06973v2","url_pdf":"https://arxiv.org/pdf/2406.06973v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"rwkv-clip-a-robust-vision-language","repo_url":"https://github.com/deepglint/rwkv-clip","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"rwkv-clip-a-robust-vision-language","repo_url":"https://github.com/MindCode-4/code-4/tree/main/rwkv","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null}],"tasks":[{"task_slug":"image-text-retrieval","task_name":"Image-text Retrieval"},{"task_slug":"representation-learning","task_name":"Representation Learning"},{"task_slug":"text-retrieval","task_name":"Text Retrieval"},{"task_slug":"zero-shot-learning","task_name":"Zero-Shot Learning"},{"task_slug":null,"task_name":"zero-shot-classification"}],"methods":[{"method_slug":"clip","method_name":"CLIP"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2406.06973","atlas_url":"https://app.syntology.ai/?focus=2406.06973","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.06973"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/deepglint/rwkv-clip","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/MindCode-4/code-4/tree/main/rwkv","reach":null},{"provenance":"deterministic:regex_extraction","url":"https://github.com/deepglint/RWKV-CLIP","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":3,"ran_fixture":1,"ran":3,"unverified":7},"by_repo_kind":{"official":{"samples":14,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"98f385d847636a3e","entry":"basic_clean","repo":"deepglint/RWKV-CLIP","repo_kind":"official","path":"model/open_clip/tokenizer.py","file_url":"https://github.com/deepglint/RWKV-CLIP/blob/HEAD/model/open_clip/tokenizer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"98f385d847636a3e"}},{"code_sha256_prefix":"87577b3ff9d32712","entry":"drop_path","repo":"deepglint/RWKV-CLIP","repo_kind":"official","path":"model/utils_vision_rwkv/drop.py","file_url":"https://github.com/deepglint/RWKV-CLIP/blob/HEAD/model/utils_vision_rwkv/drop.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"87577b3ff9d32712"}},{"code_sha256_prefix":"d919ae32e5e4e616","entry":"get_pairs","repo":"deepglint/RWKV-CLIP","repo_kind":"official","path":"model/open_clip/tokenizer.py","file_url":"https://github.com/deepglint/RWKV-CLIP/blob/HEAD/model/open_clip/tokenizer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d919ae32e5e4e616"}},{"code_sha256_prefix":"ff9bfb497bc486af","entry":"image_transform","repo":"deepglint/RWKV-CLIP","repo_kind":"official","path":"model/open_clip/transform.py","file_url":"https://github.com/deepglint/RWKV-CLIP/blob/HEAD/model/open_clip/transform.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ff9bfb497bc486af"}},{"code_sha256_prefix":"59b9fe12dfcd73e5","entry":"load_model_configs","repo":"deepglint/RWKV-CLIP","repo_kind":"official","path":"model_config/utils_notebook.py","file_url":"https://github.com/deepglint/RWKV-CLIP/blob/HEAD/model_config/utils_notebook.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"59b9fe12dfcd73e5"}},{"code_sha256_prefix":"f159f1ed836af624","entry":"load_model_weight","repo":"deepglint/RWKV-CLIP","repo_kind":"official","path":"model/utils.py","file_url":"https://github.com/deepglint/RWKV-CLIP/blob/HEAD/model/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f159f1ed836af624"}},{"code_sha256_prefix":"9542161e9640b858","entry":"whitespace_clean","repo":"deepglint/RWKV-CLIP","repo_kind":"official","path":"model/open_clip/tokenizer.py","file_url":"https://github.com/deepglint/RWKV-CLIP/blob/HEAD/model/open_clip/tokenizer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9542161e9640b858"}},{"code_sha256_prefix":"4a3350dc35016870","entry":"WarperCLIP_V_T_RWKV_method","repo":"deepglint/RWKV-CLIP","repo_kind":"official","path":"model/utils.py","file_url":"https://github.com/deepglint/RWKV-CLIP/blob/HEAD/model/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4a3350dc35016870"}},{"code_sha256_prefix":"cd7722094b3b2370","entry":"WarperCLIP_V_T_RWKV_text_change_head","repo":"deepglint/RWKV-CLIP","repo_kind":"official","path":"model/utils.py","file_url":"https://github.com/deepglint/RWKV-CLIP/blob/HEAD/model/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cd7722094b3b2370"}},{"code_sha256_prefix":"4d278aa7e3a2f51e","entry":"compute_retrieval","repo":"deepglint/RWKV-CLIP","repo_kind":"official","path":"text_image_retrieval.py","file_url":"https://github.com/deepglint/RWKV-CLIP/blob/HEAD/text_image_retrieval.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4d278aa7e3a2f51e"}},{"code_sha256_prefix":"9da98aebf2dd1b3a","entry":"create_dali_pipeline","repo":"deepglint/RWKV-CLIP","repo_kind":"official","path":"dataloaders/imagenet.py","file_url":"https://github.com/deepglint/RWKV-CLIP/blob/HEAD/dataloaders/imagenet.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9da98aebf2dd1b3a"}},{"code_sha256_prefix":"ddcbd45e940484ee","entry":"gather_features","repo":"deepglint/RWKV-CLIP","repo_kind":"official","path":"loss.py","file_url":"https://github.com/deepglint/RWKV-CLIP/blob/HEAD/loss.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ddcbd45e940484ee"}},{"code_sha256_prefix":"f9b03b79fff32f11","entry":"generate_init_weight_V6","repo":"deepglint/RWKV-CLIP","repo_kind":"official","path":"model/Text_rwkv.py","file_url":"https://github.com/deepglint/RWKV-CLIP/blob/HEAD/model/Text_rwkv.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f9b03b79fff32f11"}},{"code_sha256_prefix":"5eaaaa9625688979","entry":"get_image_feature","repo":"deepglint/RWKV-CLIP","repo_kind":"official","path":"text_image_retrieval.py","file_url":"https://github.com/deepglint/RWKV-CLIP/blob/HEAD/text_image_retrieval.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5eaaaa9625688979"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}