{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/language-only-efficient-training-of-zero-shot","title":"Language-only Efficient Training of Zero-shot Composed Image Retrieval","arxiv_id":"2312.01998","date":"2023-12-04","proceeding":null,"authors":["Geonmo Gu","Sanghyuk Chun","Wonjae Kim","Yoohoon Kang","Sangdoo Yun"],"abstract":"Composed image retrieval (CIR) task takes a composed query of image and text, aiming to search relative images for both conditions. Conventional CIR approaches need a training dataset composed of triplets of query image, query text, and target image, which is very expensive to collect. Several recent works have worked on the zero-shot (ZS) CIR paradigm to tackle the issue without using pre-collected triplets. However, the existing ZS-CIR methods show limited backbone scalability and generalizability due to the lack of diversity of the input texts during training. We propose a novel CIR framework, only using language for its training. Our LinCIR (Language-only training for CIR) can be trained only with text datasets by a novel self-supervision named self-masking projection (SMP). We project the text latent embedding to the token embedding space and construct a new text by replacing the keyword tokens of the original text. Then, we let the new and original texts have the same latent embedding vector. With this simple strategy, LinCIR is surprisingly efficient and highly effective; LinCIR with CLIP ViT-G backbone is trained in 48 minutes and shows the best ZS-CIR performances on four different CIR benchmarks, CIRCO, GeneCIS, FashionIQ, and CIRR, even outperforming supervised method on FashionIQ. Code is available at https://github.com/navervision/lincir","url_abs":"https://arxiv.org/abs/2312.01998v2","url_pdf":"https://arxiv.org/pdf/2312.01998v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"language-only-efficient-training-of-zero-shot","repo_url":"https://github.com/navervision/lincir","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}}],"tasks":[{"task_slug":"image-retrieval","task_name":"Image Retrieval"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"zero-shot-composed-image-retrieval-zs-cir","task_name":"Zero-Shot Composed Image Retrieval (ZS-CIR)"}],"methods":[{"method_slug":"clip","method_name":"CLIP"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/zero-shot-composed-image-retrieval-zs-cir-on","task":"Zero-Shot Composed Image Retrieval (ZS-CIR)","dataset":"CIRCO","model":"LinCIR (CLIP G/14)","rank_in_archive_order":22,"of":43,"metrics":{"mAP@10":"21.01"},"uses_additional_data":false},{"leaderboard":"/sota/zero-shot-composed-image-retrieval-zs-cir-on","task":"Zero-Shot Composed Image Retrieval (ZS-CIR)","dataset":"CIRCO","model":"LinCIR (CLIP L/14)","rank_in_archive_order":33,"of":43,"metrics":{"mAP@10":"13.58"},"uses_additional_data":false},{"leaderboard":"/sota/zero-shot-composed-image-retrieval-zs-cir-on-1","task":"Zero-Shot Composed Image Retrieval (ZS-CIR)","dataset":"CIRR","model":"LinCIR (CLIP G/14)","rank_in_archive_order":26,"of":47,"metrics":{"R@5":"64.72"},"uses_additional_data":false},{"leaderboard":"/sota/zero-shot-composed-image-retrieval-zs-cir-on-1","task":"Zero-Shot Composed Image Retrieval (ZS-CIR)","dataset":"CIRR","model":"LinCIR (CLIP L/14)","rank_in_archive_order":44,"of":47,"metrics":{"R@5":"53.25"},"uses_additional_data":false},{"leaderboard":"/sota/zero-shot-composed-image-retrieval-zs-cir-on-2","task":"Zero-Shot Composed Image Retrieval (ZS-CIR)","dataset":"Fashion IQ","model":"LinCIR (CLIP G/14)","rank_in_archive_order":2,"of":41,"metrics":{"(Recall@10+Recall@50)/2":"55.40"},"uses_additional_data":false},{"leaderboard":"/sota/zero-shot-composed-image-retrieval-zs-cir-on-2","task":"Zero-Shot Composed Image Retrieval (ZS-CIR)","dataset":"Fashion IQ","model":"LinCIR (CLIP L/14)","rank_in_archive_order":33,"of":41,"metrics":{"(Recall@10+Recall@50)/2":"36.39"},"uses_additional_data":false},{"leaderboard":"/sota/zero-shot-composed-image-retrieval-zs-cir-on-11","task":"Zero-Shot Composed Image Retrieval (ZS-CIR)","dataset":"GeneCIS","model":"LinCIR (CLIP G/14)","rank_in_archive_order":6,"of":11,"metrics":{" A-R@1":"13.7"},"uses_additional_data":false},{"leaderboard":"/sota/zero-shot-composed-image-retrieval-zs-cir-on-11","task":"Zero-Shot Composed Image Retrieval (ZS-CIR)","dataset":"GeneCIS","model":"LinCIR (CLIP L/14)","rank_in_archive_order":8,"of":11,"metrics":{" A-R@1":"12.2"},"uses_additional_data":false},{"leaderboard":"/sota/zero-shot-composed-image-retrieval-zs-cir-on-5","task":"Zero-Shot Composed Image Retrieval (ZS-CIR)","dataset":"ImageNet","model":"LinCIR (CLIP L/14)","rank_in_archive_order":3,"of":11,"metrics":{"Average Recall":"21.64"},"uses_additional_data":false},{"leaderboard":"/sota/zero-shot-composed-image-retrieval-zs-cir-on-6","task":"Zero-Shot Composed Image Retrieval (ZS-CIR)","dataset":"ImageNet-R","model":"LinCIR (CLIP L/14)","rank_in_archive_order":6,"of":20,"metrics":{"(Recall@10+Recall@50)/2":"21.64"},"uses_additional_data":false}],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2312.01998","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.01998"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/navervision/lincir","reach":{"status":"ok","spdx":"NOASSERTION"}}],"summary":{"ran":2,"ran_honours":1,"unverified":5},"by_repo_kind":{"official":{"samples":8,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":8,"samples":[{"code_sha256_prefix":"09872c29f7fe0d5f","entry":"clean_caption","repo":"navervision/lincir","repo_kind":"official","path":"loader.py","file_url":"https://github.com/navervision/lincir/blob/HEAD/loader.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"09872c29f7fe0d5f"}},{"code_sha256_prefix":"b6c1f837e6faaeba","entry":"collate_fn","repo":"navervision/lincir","repo_kind":"official","path":"data_utils.py","file_url":"https://github.com/navervision/lincir/blob/HEAD/data_utils.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b6c1f837e6faaeba"}},{"code_sha256_prefix":"9476d6cebb3ecfb9","entry":"targetpad_transform","repo":"navervision/lincir","repo_kind":"official","path":"data_utils.py","file_url":"https://github.com/navervision/lincir/blob/HEAD/data_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9476d6cebb3ecfb9"}},{"code_sha256_prefix":"2ceb37f80156acbe","entry":"build_text_encoder","repo":"navervision/lincir","repo_kind":"official","path":"models.py","file_url":"https://github.com/navervision/lincir/blob/HEAD/models.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"2ceb37f80156acbe"}},{"code_sha256_prefix":"f96edb7ca8fe54fe","entry":"contrastive_loss","repo":"navervision/lincir","repo_kind":"official","path":"utils.py","file_url":"https://github.com/navervision/lincir/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f96edb7ca8fe54fe"}},{"code_sha256_prefix":"850ce1bfba15d924","entry":"encode_with_pseudo_tokens_HF","repo":"navervision/lincir","repo_kind":"official","path":"encode_with_pseudo_tokens.py","file_url":"https://github.com/navervision/lincir/blob/HEAD/encode_with_pseudo_tokens.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"850ce1bfba15d924"}},{"code_sha256_prefix":"305c8808a50869a9","entry":"extract_keywords","repo":"navervision/lincir","repo_kind":"official","path":"loader.py","file_url":"https://github.com/navervision/lincir/blob/HEAD/loader.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"305c8808a50869a9"}},{"code_sha256_prefix":"abf687befc030638","entry":"extract_keywords_spacy","repo":"navervision/lincir","repo_kind":"official","path":"loader.py","file_url":"https://github.com/navervision/lincir/blob/HEAD/loader.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"abf687befc030638"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}