{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/a-simple-latent-diffusion-approach-for","title":"A Simple Latent Diffusion Approach for Panoptic Segmentation and Mask Inpainting","arxiv_id":"2401.10227","date":"2024-01-18","proceeding":null,"authors":["Wouter Van Gansbeke","Bert de Brabandere"],"abstract":"Panoptic and instance segmentation networks are often trained with specialized object detection modules, complex loss functions, and ad-hoc post-processing steps to manage the permutation-invariance of the instance masks. This work builds upon Stable Diffusion and proposes a latent diffusion approach for panoptic segmentation, resulting in a simple architecture that omits these complexities. Our training consists of two steps: (1) training a shallow autoencoder to project the segmentation masks to latent space; (2) training a diffusion model to allow image-conditioned sampling in latent space. This generative approach unlocks the exploration of mask completion or inpainting. The experimental validation on COCO and ADE20k yields strong segmentation results. Finally, we demonstrate our model's adaptability to multi-tasking by introducing learnable task embeddings.","url_abs":"https://arxiv.org/abs/2401.10227v2","url_pdf":"https://arxiv.org/pdf/2401.10227v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"a-simple-latent-diffusion-approach-for","repo_url":"https://github.com/segments-ai/latent-diffusion-segmentation","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}}],"tasks":[{"task_slug":"instance-segmentation","task_name":"Instance Segmentation"},{"task_slug":"interactive-segmentation","task_name":"Interactive Segmentation"},{"task_slug":"object-detection","task_name":"Object Detection"},{"task_slug":"panoptic-segmentation","task_name":"Panoptic Segmentation"},{"task_slug":"segmentation","task_name":"Segmentation"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"},{"task_slug":"object-detection-1","task_name":"object-detection"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2401.10227","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2401.10227"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/segments-ai/latent-diffusion-segmentation","reach":{"status":"ok","spdx":"NOASSERTION"}}],"summary":{"ran":5,"unverified":2},"by_repo_kind":{"official":{"samples":7,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":7,"samples":[{"code_sha256_prefix":"285d6b71a60bea8d","entry":"get_optim","repo":"segments-ai/latent-diffusion-segmentation","repo_kind":"official","path":"ldmseg/trainers/optim.py","file_url":"https://github.com/segments-ai/latent-diffusion-segmentation/blob/HEAD/ldmseg/trainers/optim.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"285d6b71a60bea8d"}},{"code_sha256_prefix":"00c51b6b8d648c08","entry":"get_optim_general","repo":"segments-ai/latent-diffusion-segmentation","repo_kind":"official","path":"ldmseg/trainers/optim.py","file_url":"https://github.com/segments-ai/latent-diffusion-segmentation/blob/HEAD/ldmseg/trainers/optim.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"00c51b6b8d648c08"}},{"code_sha256_prefix":"3562bf8930bae102","entry":"get_optim_unet","repo":"segments-ai/latent-diffusion-segmentation","repo_kind":"official","path":"ldmseg/trainers/optim.py","file_url":"https://github.com/segments-ai/latent-diffusion-segmentation/blob/HEAD/ldmseg/trainers/optim.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"3562bf8930bae102"}},{"code_sha256_prefix":"016b2be6df1fe0ac","entry":"get_table","repo":"segments-ai/latent-diffusion-segmentation","repo_kind":"official","path":"ldmseg/evaluations/panoptic_evaluation.py","file_url":"https://github.com/segments-ai/latent-diffusion-segmentation/blob/HEAD/ldmseg/evaluations/panoptic_evaluation.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"016b2be6df1fe0ac"}},{"code_sha256_prefix":"ee10bcf387ebc6e5","entry":"get_table","repo":"segments-ai/latent-diffusion-segmentation","repo_kind":"official","path":"ldmseg/evaluations/panoptic_evaluation_agnostic.py","file_url":"https://github.com/segments-ai/latent-diffusion-segmentation/blob/HEAD/ldmseg/evaluations/panoptic_evaluation_agnostic.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"ee10bcf387ebc6e5"}},{"code_sha256_prefix":"f67634d15b336a0e","entry":"get_image_descriptor_model","repo":"segments-ai/latent-diffusion-segmentation","repo_kind":"official","path":"ldmseg/models/descriptors.py","file_url":"https://github.com/segments-ai/latent-diffusion-segmentation/blob/HEAD/ldmseg/models/descriptors.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"f67634d15b336a0e"}},{"code_sha256_prefix":"9411200fcda3bcd3","entry":"pq_compute","repo":"segments-ai/latent-diffusion-segmentation","repo_kind":"official","path":"ldmseg/evaluations/panoptic_evaluation_agnostic.py","file_url":"https://github.com/segments-ai/latent-diffusion-segmentation/blob/HEAD/ldmseg/evaluations/panoptic_evaluation_agnostic.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"9411200fcda3bcd3"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}