{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/language-driven-semantic-segmentation-1","title":"Language-driven Semantic Segmentation","arxiv_id":"2201.03546","date":"2022-01-10","proceeding":"ICLR 2022 4","authors":["Boyi Li","Kilian Q. Weinberger","Serge Belongie","Vladlen Koltun","René Ranftl"],"abstract":"We present LSeg, a novel model for language-driven semantic image segmentation. LSeg uses a text encoder to compute embeddings of descriptive input labels (e.g., \"grass\" or \"building\") together with a transformer-based image encoder that computes dense per-pixel embeddings of the input image. The image encoder is trained with a contrastive objective to align pixel embeddings to the text embedding of the corresponding semantic class. The text embeddings provide a flexible label representation in which semantically similar labels map to similar regions in the embedding space (e.g., \"cat\" and \"furry\"). This allows LSeg to generalize to previously unseen categories at test time, without retraining or even requiring a single additional training sample. We demonstrate that our approach achieves highly competitive zero-shot performance compared to existing zero- and few-shot semantic segmentation methods, and even matches the accuracy of traditional segmentation algorithms when a fixed label set is provided. Code and demo are available at https://github.com/isl-org/lang-seg.","url_abs":"https://arxiv.org/abs/2201.03546v2","url_pdf":"https://arxiv.org/pdf/2201.03546v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"language-driven-semantic-segmentation-1","repo_url":"https://github.com/isl-org/lang-seg","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"descriptive","task_name":"Descriptive"},{"task_slug":"few-shot-image-segmentation","task_name":"Few-Shot Semantic Segmentation"},{"task_slug":"image-segmentation","task_name":"Image Segmentation"},{"task_slug":"segmentation","task_name":"Segmentation"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"},{"task_slug":"zero-shot-segmentation","task_name":"Zero Shot Segmentation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/few-shot-semantic-segmentation-on-fss-1000","task":"Few-Shot Semantic Segmentation","dataset":"FSS-1000","model":"LSeg","rank_in_archive_order":1,"of":3,"metrics":{"Mean IoU":"87.8"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2201.03546","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2201.03546"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/isl-org/lang-seg","reach":null}],"summary":{"ran_fixture":1,"ran_honours":2},"by_repo_kind":{"official":{"samples":2,"ran":2,"repositories":1},"community":{"samples":1,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":1,"samples":[{"code_sha256_prefix":"cd8fd543b1b4b1fe","entry":"get_new_mask_pallete","repo":"isl-org/lang-seg","repo_kind":"official","path":"lseg_app.py","file_url":"https://github.com/isl-org/lang-seg/blob/HEAD/lseg_app.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cd8fd543b1b4b1fe"}},{"code_sha256_prefix":"9803abda3688453c","entry":"get_new_pallete","repo":"krrish94/lseg-minimal","repo_kind":"community","path":"examples/lseg_openset_seg.py","file_url":"https://github.com/krrish94/lseg-minimal/blob/HEAD/examples/lseg_openset_seg.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9803abda3688453c"}},{"code_sha256_prefix":"0fc141584bd295d9","entry":"get_new_pallete","repo":"isl-org/lang-seg","repo_kind":"official","path":"lseg_app.py","file_url":"https://github.com/isl-org/lang-seg/blob/HEAD/lseg_app.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0fc141584bd295d9"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}