{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2601-06891","title":"CLIMP: Contrastive Language-Image Mamba Pretraining","arxiv_id":"2601.06891","date":"2026-01-11","proceeding":null,"authors":["Nimrod Shabtay","Itamar Zimerman","Eli Schwartz","Raja Giryes"],"abstract":"Contrastive Language-Image Pre-training (CLIP) relies on Vision Transformers whose attention mechanism is susceptible to spurious correlations, and scales quadratically with resolution. To address these limitations, We present CLIMP, the first fully Mamba-based contrastive vision-language model that replaces both the vision and text encoders with Mamba. The new architecture encodes sequential structure in both vision and language, with VMamba capturing visual spatial inductive biases, reducing reliance on spurious correlations and producing an embedding space favorable for cross-modal retrieval and out-of-distribution robustness-surpassing OpenAI's CLIP-ViT-B by 7.5% on ImageNet-O. CLIMP naturally supports variable input resolutions without positional encoding interpolation or specialized training, achieving up to 6.6% higher retrieval accuracy at 16x training resolution while using 5x less memory and 1.8x fewer FLOPs. The autoregressive text encoder further overcomes CLIP's fixed context limitation, enabling dense captioning retrieval. Our findings suggest that Mamba exhibits advantageous properties for vision-language learning, making it a compelling alternative to Transformer-based CLIP.The code and models are publicly available at https://github.com/NimrodShabtay/CLIMP}","url_abs":"https://arxiv.org/abs/2601.06891","url_pdf":"https://arxiv.org/pdf/2601.06891","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2601.06891","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2601.06891"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/NimrodShabtay/CLIMP","reach":{"status":"ok"}}],"summary":{"unverified":17},"by_repo_kind":{"found_in_text":{"samples":17,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":17,"samples":[{"code_sha256_prefix":"23e45b8dd25f1b84","entry":"batchify","repo":"NimrodShabtay/CLIMP","repo_kind":"found_in_text","path":"eval_nocaps.py","file_url":"https://github.com/NimrodShabtay/CLIMP/blob/HEAD/eval_nocaps.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"23e45b8dd25f1b84"}},{"code_sha256_prefix":"19d037c724c38dbb","entry":"chunk_scan_ref","repo":"NimrodShabtay/CLIMP","repo_kind":"found_in_text","path":"models/vmamba/mamba2/ssd_chunk_scan.py","file_url":"https://github.com/NimrodShabtay/CLIMP/blob/HEAD/models/vmamba/mamba2/ssd_chunk_scan.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"19d037c724c38dbb"}},{"code_sha256_prefix":"44d8f6aca6af7092","entry":"chunk_state_ref","repo":"NimrodShabtay/CLIMP","repo_kind":"found_in_text","path":"models/vmamba/mamba2/ssd_chunk_state.py","file_url":"https://github.com/NimrodShabtay/CLIMP/blob/HEAD/models/vmamba/mamba2/ssd_chunk_state.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"44d8f6aca6af7092"}},{"code_sha256_prefix":"f52bda65e1c22ab1","entry":"collate_fn","repo":"NimrodShabtay/CLIMP","repo_kind":"found_in_text","path":"eval_nocaps.py","file_url":"https://github.com/NimrodShabtay/CLIMP/blob/HEAD/eval_nocaps.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f52bda65e1c22ab1"}},{"code_sha256_prefix":"e8dc1915d58c8e3e","entry":"cross_merge_fwd","repo":"NimrodShabtay/CLIMP","repo_kind":"found_in_text","path":"models/vmamba/csm_triton.py","file_url":"https://github.com/NimrodShabtay/CLIMP/blob/HEAD/models/vmamba/csm_triton.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e8dc1915d58c8e3e"}},{"code_sha256_prefix":"ff7009b253890fe2","entry":"cross_scan1b1_fwd","repo":"NimrodShabtay/CLIMP","repo_kind":"found_in_text","path":"models/vmamba/csm_triton.py","file_url":"https://github.com/NimrodShabtay/CLIMP/blob/HEAD/models/vmamba/csm_triton.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ff7009b253890fe2"}},{"code_sha256_prefix":"58ffc4548a15baa1","entry":"cross_scan_fwd","repo":"NimrodShabtay/CLIMP","repo_kind":"found_in_text","path":"models/vmamba/csm_triton.py","file_url":"https://github.com/NimrodShabtay/CLIMP/blob/HEAD/models/vmamba/csm_triton.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"58ffc4548a15baa1"}},{"code_sha256_prefix":"9a3adc8e6de35ac1","entry":"flops_selective_scan_fn","repo":"NimrodShabtay/CLIMP","repo_kind":"found_in_text","path":"models/vmamba/csms6s.py","file_url":"https://github.com/NimrodShabtay/CLIMP/blob/HEAD/models/vmamba/csms6s.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9a3adc8e6de35ac1"}},{"code_sha256_prefix":"f7a500c0f491fa7a","entry":"init_to_zero","repo":"NimrodShabtay/CLIMP","repo_kind":"found_in_text","path":"models/vmamba/mamba2/ssd_bmm.py","file_url":"https://github.com/NimrodShabtay/CLIMP/blob/HEAD/models/vmamba/mamba2/ssd_bmm.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f7a500c0f491fa7a"}},{"code_sha256_prefix":"993310f8aed36f6d","entry":"layer_norm_ref","repo":"NimrodShabtay/CLIMP","repo_kind":"found_in_text","path":"models/vmamba/mamba2/layer_norm.py","file_url":"https://github.com/NimrodShabtay/CLIMP/blob/HEAD/models/vmamba/mamba2/layer_norm.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"993310f8aed36f6d"}},{"code_sha256_prefix":"571ef30fa651077b","entry":"recall_at_k","repo":"NimrodShabtay/CLIMP","repo_kind":"found_in_text","path":"eval_nocaps.py","file_url":"https://github.com/NimrodShabtay/CLIMP/blob/HEAD/eval_nocaps.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"571ef30fa651077b"}},{"code_sha256_prefix":"8ddbc28ea965c11f","entry":"rms_norm_ref","repo":"NimrodShabtay/CLIMP","repo_kind":"found_in_text","path":"models/vmamba/mamba2/layer_norm.py","file_url":"https://github.com/NimrodShabtay/CLIMP/blob/HEAD/models/vmamba/mamba2/layer_norm.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8ddbc28ea965c11f"}},{"code_sha256_prefix":"019b69251d0aed68","entry":"rms_norm_ref","repo":"NimrodShabtay/CLIMP","repo_kind":"found_in_text","path":"models/vmamba/mamba2/layernorm_gated.py","file_url":"https://github.com/NimrodShabtay/CLIMP/blob/HEAD/models/vmamba/mamba2/layernorm_gated.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"019b69251d0aed68"}},{"code_sha256_prefix":"85b4b58886782b6b","entry":"selective_scan_bwd","repo":"NimrodShabtay/CLIMP","repo_kind":"found_in_text","path":"models/vmamba/mamba2/ssd_combined.py","file_url":"https://github.com/NimrodShabtay/CLIMP/blob/HEAD/models/vmamba/mamba2/ssd_combined.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"85b4b58886782b6b"}},{"code_sha256_prefix":"0154022b08d93f9a","entry":"selective_scan_fn","repo":"NimrodShabtay/CLIMP","repo_kind":"found_in_text","path":"models/vmamba/csms6s.py","file_url":"https://github.com/NimrodShabtay/CLIMP/blob/HEAD/models/vmamba/csms6s.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0154022b08d93f9a"}},{"code_sha256_prefix":"82e4cdd256f30ead","entry":"selective_scan_torch","repo":"NimrodShabtay/CLIMP","repo_kind":"found_in_text","path":"models/vmamba/csms6s.py","file_url":"https://github.com/NimrodShabtay/CLIMP/blob/HEAD/models/vmamba/csms6s.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"82e4cdd256f30ead"}},{"code_sha256_prefix":"d4c09b0a322e7cc0","entry":"selective_state_update_ref","repo":"NimrodShabtay/CLIMP","repo_kind":"found_in_text","path":"models/vmamba/mamba2/selective_state_update.py","file_url":"https://github.com/NimrodShabtay/CLIMP/blob/HEAD/models/vmamba/mamba2/selective_state_update.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d4c09b0a322e7cc0"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.CV","source":"arxiv_2026.jsonl"},"syntology_extracted_results":null}