{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/scaling-language-image-pre-training-via","title":"Scaling Language-Image Pre-training via Masking","arxiv_id":"2212.00794","date":"2022-12-01","proceeding":"CVPR 2023 1","authors":["Yanghao Li","Haoqi Fan","Ronghang Hu","Christoph Feichtenhofer","Kaiming He"],"abstract":"We present Fast Language-Image Pre-training (FLIP), a simple and more efficient method for training CLIP. Our method randomly masks out and removes a large portion of image patches during training. Masking allows us to learn from more image-text pairs given the same wall-clock time and contrast more samples per iteration with similar memory footprint. It leads to a favorable trade-off between accuracy and training time. In our experiments on 400 million image-text pairs, FLIP improves both accuracy and speed over the no-masking baseline. On a large diversity of downstream tasks, FLIP dominantly outperforms the CLIP counterparts trained on the same data. Facilitated by the speedup, we explore the scaling behavior of increasing the model size, data size, or training length, and report encouraging results and comparisons. We hope that our work will foster future research on scaling vision-language learning.","url_abs":"https://arxiv.org/abs/2212.00794v2","url_pdf":"https://arxiv.org/pdf/2212.00794v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"scaling-language-image-pre-training-via","repo_url":"https://github.com/facebookresearch/flip","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"jax","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"scaling-language-image-pre-training-via","repo_url":"https://github.com/ale9806/open_clip_with_biomedica","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"scaling-language-image-pre-training-via","repo_url":"https://github.com/lucidrains/CoCa-pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"scaling-language-image-pre-training-via","repo_url":"https://github.com/lucidrains/mammut-pytorch","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"scaling-language-image-pre-training-via","repo_url":"https://github.com/mlfoundations/open_clip","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"scaling-language-image-pre-training-via","repo_url":"https://github.com/lucidrains/x-clip","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"diversity","task_name":"Diversity"}],"methods":[{"method_slug":"clip","method_name":"CLIP"},{"method_slug":"flip","method_name":"FLIP"},{"method_slug":"speed","method_name":"SPEED"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2212.00794","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2212.00794"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mlfoundations/open_clip","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/flip","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/lucidrains/CoCa-pytorch","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ale9806/open_clip_with_biomedica","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/lucidrains/mammut-pytorch","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/lucidrains/x-clip","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_honours":2,"ran_draft_wrong":3,"ran_violates":5,"unverified":4},"by_repo_kind":{"listed":{"samples":14,"ran":10,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"7f1040f5e3991d5e","entry":"identity","repo":"lucidrains/x-clip","repo_kind":"listed","path":"x_clip/x_clip.py","file_url":"https://github.com/lucidrains/x-clip/blob/HEAD/x_clip/x_clip.py","link_basis":"plan_row","language":"python","status":"ran_honours","verification_level":2,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7f1040f5e3991d5e"}},{"code_sha256_prefix":"98f385d847636a3e","entry":"basic_clean","repo":"lucidrains/x-clip","repo_kind":"listed","path":"x_clip/tokenizer.py","file_url":"https://github.com/lucidrains/x-clip/blob/HEAD/x_clip/tokenizer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"98f385d847636a3e"}},{"code_sha256_prefix":"60fff7c3c400d7ff","entry":"default","repo":"lucidrains/x-clip","repo_kind":"listed","path":"x_clip/x_clip.py","file_url":"https://github.com/lucidrains/x-clip/blob/HEAD/x_clip/x_clip.py","link_basis":"plan_row","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"60fff7c3c400d7ff"}},{"code_sha256_prefix":"1c0f269ae9673b08","entry":"default","repo":"lucidrains/x-clip","repo_kind":"listed","path":"x_clip/visual_ssl.py","file_url":"https://github.com/lucidrains/x-clip/blob/HEAD/x_clip/visual_ssl.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1c0f269ae9673b08"}},{"code_sha256_prefix":"aa5486a3650902d8","entry":"exists","repo":"lucidrains/x-clip","repo_kind":"listed","path":"x_clip/x_clip.py","file_url":"https://github.com/lucidrains/x-clip/blob/HEAD/x_clip/x_clip.py","link_basis":"plan_row","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"aa5486a3650902d8"}},{"code_sha256_prefix":"4daa3e876f494d75","entry":"flatten","repo":"lucidrains/x-clip","repo_kind":"listed","path":"x_clip/visual_ssl.py","file_url":"https://github.com/lucidrains/x-clip/blob/HEAD/x_clip/visual_ssl.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4daa3e876f494d75"}},{"code_sha256_prefix":"e33ada12822bb56c","entry":"get_mask_subset_with_prob","repo":"lucidrains/x-clip","repo_kind":"listed","path":"x_clip/mlm.py","file_url":"https://github.com/lucidrains/x-clip/blob/HEAD/x_clip/mlm.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e33ada12822bb56c"}},{"code_sha256_prefix":"5573a79bdc0a0a09","entry":"mask_with_tokens","repo":"lucidrains/x-clip","repo_kind":"listed","path":"x_clip/mlm.py","file_url":"https://github.com/lucidrains/x-clip/blob/HEAD/x_clip/mlm.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5573a79bdc0a0a09"}},{"code_sha256_prefix":"0a27e49623646a01","entry":"prob_mask_like","repo":"lucidrains/x-clip","repo_kind":"listed","path":"x_clip/mlm.py","file_url":"https://github.com/lucidrains/x-clip/blob/HEAD/x_clip/mlm.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0a27e49623646a01"}},{"code_sha256_prefix":"9542161e9640b858","entry":"whitespace_clean","repo":"lucidrains/x-clip","repo_kind":"listed","path":"x_clip/tokenizer.py","file_url":"https://github.com/lucidrains/x-clip/blob/HEAD/x_clip/tokenizer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9542161e9640b858"}},{"code_sha256_prefix":"1022a3b9a987f9af","entry":"all_gather_variable_dim","repo":"lucidrains/x-clip","repo_kind":"listed","path":"x_clip/distributed.py","file_url":"https://github.com/lucidrains/x-clip/blob/HEAD/x_clip/distributed.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1022a3b9a987f9af"}},{"code_sha256_prefix":"ca29cb49bbe686d2","entry":"get_default_aug","repo":"lucidrains/x-clip","repo_kind":"listed","path":"x_clip/visual_ssl.py","file_url":"https://github.com/lucidrains/x-clip/blob/HEAD/x_clip/visual_ssl.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ca29cb49bbe686d2"}},{"code_sha256_prefix":"b50994ad24f6d583","entry":"get_pairs","repo":"lucidrains/x-clip","repo_kind":"listed","path":"x_clip/tokenizer.py","file_url":"https://github.com/lucidrains/x-clip/blob/HEAD/x_clip/tokenizer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b50994ad24f6d583"}},{"code_sha256_prefix":"1a7818b5192e3dc5","entry":"pad_dim_to","repo":"lucidrains/x-clip","repo_kind":"listed","path":"x_clip/distributed.py","file_url":"https://github.com/lucidrains/x-clip/blob/HEAD/x_clip/distributed.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1a7818b5192e3dc5"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}