{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/unveiling-the-power-of-diffusion-features-for","title":"Where's Waldo: Diffusion Features for Personalized Segmentation and Retrieval","arxiv_id":"2405.18025","date":"2024-05-28","proceeding":null,"authors":["Dvir Samuel","Rami Ben-Ari","Matan Levy","Nir Darshan","Gal Chechik"],"abstract":"Personalized retrieval and segmentation aim to locate specific instances within a dataset based on an input image and a short description of the reference instance. While supervised methods are effective, they require extensive labeled data for training. Recently, self-supervised foundation models have been introduced to these tasks showing comparable results to supervised methods. However, a significant flaw in these models is evident: they struggle to locate a desired instance when other instances within the same class are presented. In this paper, we explore text-to-image diffusion models for these tasks. Specifically, we propose a novel approach called PDM for Personalized Features Diffusion Matching, that leverages intermediate features of pre-trained text-to-image models for personalization tasks without any additional training. PDM demonstrates superior performance on popular retrieval and segmentation benchmarks, outperforming even supervised methods. We also highlight notable shortcomings in current instance and segmentation datasets and propose new benchmarks for these tasks.","url_abs":"https://arxiv.org/abs/2405.18025v2","url_pdf":"https://arxiv.org/pdf/2405.18025v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"unveiling-the-power-of-diffusion-features-for","repo_url":"https://github.com/dvirsamuel/PDM","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"personalized-segmentation","task_name":"Personalized Segmentation"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"segmentation","task_name":"Segmentation"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2405.18025","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.18025"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/dvirsamuel/PDM","reach":{"status":"ok"}}],"summary":{"ran":8,"ran_draft_wrong":1,"unverified":3},"by_repo_kind":{"official":{"samples":12,"ran":9,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":12,"samples":[{"code_sha256_prefix":"5e61e838746ed0b4","entry":"center_crop","repo":"dvirsamuel/PDM","repo_kind":"official","path":"pdm_matching.py","file_url":"https://github.com/dvirsamuel/PDM/blob/HEAD/pdm_matching.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5e61e838746ed0b4"}},{"code_sha256_prefix":"8e7cc9e449432e62","entry":"center_crop","repo":"dvirsamuel/PDM","repo_kind":"official","path":"pdm_permir.py","file_url":"https://github.com/dvirsamuel/PDM/blob/HEAD/pdm_permir.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8e7cc9e449432e62"}},{"code_sha256_prefix":"a042902caede448b","entry":"diffusion_step","repo":"dvirsamuel/PDM","repo_kind":"official","path":"ptp_utils.py","file_url":"https://github.com/dvirsamuel/PDM/blob/HEAD/ptp_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a042902caede448b"}},{"code_sha256_prefix":"22f8a912eecbd608","entry":"get_correspondences","repo":"dvirsamuel/PDM","repo_kind":"official","path":"dift.py","file_url":"https://github.com/dvirsamuel/PDM/blob/HEAD/dift.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"22f8a912eecbd608"}},{"code_sha256_prefix":"e9cd6a11f36720fa","entry":"get_correspondences_seg","repo":"dvirsamuel/PDM","repo_kind":"official","path":"dift.py","file_url":"https://github.com/dvirsamuel/PDM/blob/HEAD/dift.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e9cd6a11f36720fa"}},{"code_sha256_prefix":"6ade31b66e09baf7","entry":"intersectionAndUnion","repo":"dvirsamuel/PDM","repo_kind":"official","path":"PerMIRS/eval_miou.py","file_url":"https://github.com/dvirsamuel/PDM/blob/HEAD/PerMIRS/eval_miou.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6ade31b66e09baf7"}},{"code_sha256_prefix":"e3c3bb87bbe9b4a7","entry":"latent2image","repo":"dvirsamuel/PDM","repo_kind":"official","path":"ptp_utils.py","file_url":"https://github.com/dvirsamuel/PDM/blob/HEAD/ptp_utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e3c3bb87bbe9b4a7"}},{"code_sha256_prefix":"a9c4cf6ece08bc62","entry":"load_im_into_format_from_image","repo":"dvirsamuel/PDM","repo_kind":"official","path":"pdm_permir.py","file_url":"https://github.com/dvirsamuel/PDM/blob/HEAD/pdm_permir.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a9c4cf6ece08bc62"}},{"code_sha256_prefix":"957a22837d69c59e","entry":"text_under_image","repo":"dvirsamuel/PDM","repo_kind":"official","path":"ptp_utils.py","file_url":"https://github.com/dvirsamuel/PDM/blob/HEAD/ptp_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"957a22837d69c59e"}},{"code_sha256_prefix":"cb0792c6a2ab9d57","entry":"get_masks","repo":"dvirsamuel/PDM","repo_kind":"official","path":"pdm_matching.py","file_url":"https://github.com/dvirsamuel/PDM/blob/HEAD/pdm_matching.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"cb0792c6a2ab9d57"}},{"code_sha256_prefix":"88be6be7be1b8d2f","entry":"load_im_into_format_from_path","repo":"dvirsamuel/PDM","repo_kind":"official","path":"pdm_matching.py","file_url":"https://github.com/dvirsamuel/PDM/blob/HEAD/pdm_matching.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"88be6be7be1b8d2f"}},{"code_sha256_prefix":"0ce85a70be3ada0d","entry":"load_im_into_format_from_path","repo":"dvirsamuel/PDM","repo_kind":"official","path":"pdm_permir.py","file_url":"https://github.com/dvirsamuel/PDM/blob/HEAD/pdm_permir.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0ce85a70be3ada0d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}