{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/unsupervised-semantic-correspondence-using","title":"Unsupervised Semantic Correspondence Using Stable Diffusion","arxiv_id":"2305.15581","date":"2023-05-24","proceeding":"NeurIPS 2023 11","authors":["Eric Hedlin","Gopal Sharma","Shweta Mahajan","Hossam Isack","Abhishek Kar","Andrea Tagliasacchi","Kwang Moo Yi"],"abstract":"Text-to-image diffusion models are now capable of generating images that are often indistinguishable from real images. To generate such images, these models must understand the semantics of the objects they are asked to generate. In this work we show that, without any training, one can leverage this semantic knowledge within diffusion models to find semantic correspondences - locations in multiple images that have the same semantic meaning. Specifically, given an image, we optimize the prompt embeddings of these models for maximum attention on the regions of interest. These optimized embeddings capture semantic information about the location, which can then be transferred to another image. By doing so we obtain results on par with the strongly supervised state of the art on the PF-Willow dataset and significantly outperform (20.9% relative for the SPair-71k dataset) any existing weakly or unsupervised method on PF-Willow, CUB-200 and SPair-71k datasets.","url_abs":"https://arxiv.org/abs/2305.15581v2","url_pdf":"https://arxiv.org/pdf/2305.15581v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"unsupervised-semantic-correspondence-using","repo_url":"https://github.com/ubc-vision/LDM_correspondences","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"semantic-correspondence","task_name":"Semantic correspondence"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/semantic-correspondence-on-cub-200-2011","task":"Semantic correspondence","dataset":"CUB-200-2011","model":"LDM Correspondences","rank_in_archive_order":1,"of":1,"metrics":{"Mean PCK@0.05":"61.6","Mean PCK@0.1":"77.5"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-correspondence-on-pf-willow","task":"Semantic correspondence","dataset":"PF-WILLOW","model":"LDMCorrespondences","rank_in_archive_order":1,"of":8,"metrics":{"PCK":"84.3"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-correspondence-on-spair-71k","task":"Semantic correspondence","dataset":"SPair-71k","model":"LDMCorrespondences","rank_in_archive_order":18,"of":22,"metrics":{"PCK":"45.4"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2305.15581","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.15581"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ubc-vision/LDM_correspondences","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_violates":1,"ran_draft_wrong":2,"ran_honours":1,"unverified":8},"by_repo_kind":{"official":{"samples":12,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"b1f9cebf0d177140","entry":"boolean_string","repo":"ubc-vision/LDM_correspondences","repo_kind":"official","path":"utils/utils.py","file_url":"https://github.com/ubc-vision/LDM_correspondences/blob/HEAD/utils/utils.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b1f9cebf0d177140"}},{"code_sha256_prefix":"caadc5c4b7627e0c","entry":"init_latent","repo":"ubc-vision/LDM_correspondences","repo_kind":"official","path":"utils/ptp_utils.py","file_url":"https://github.com/ubc-vision/LDM_correspondences/blob/HEAD/utils/ptp_utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"caadc5c4b7627e0c"}},{"code_sha256_prefix":"e3c3bb87bbe9b4a7","entry":"latent2image","repo":"ubc-vision/LDM_correspondences","repo_kind":"official","path":"utils/ptp_utils.py","file_url":"https://github.com/ubc-vision/LDM_correspondences/blob/HEAD/utils/ptp_utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e3c3bb87bbe9b4a7"}},{"code_sha256_prefix":"0230471a58926463","entry":"load_512","repo":"ubc-vision/LDM_correspondences","repo_kind":"official","path":"utils/optimize_token.py","file_url":"https://github.com/ubc-vision/LDM_correspondences/blob/HEAD/utils/optimize_token.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0230471a58926463"}},{"code_sha256_prefix":"a9127186bb55b3f4","entry":"diffusion_step","repo":"ubc-vision/LDM_correspondences","repo_kind":"official","path":"utils/ptp_utils.py","file_url":"https://github.com/ubc-vision/LDM_correspondences/blob/HEAD/utils/ptp_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a9127186bb55b3f4"}},{"code_sha256_prefix":"d743428e092c7a2d","entry":"find_knn","repo":"ubc-vision/LDM_correspondences","repo_kind":"official","path":"eval/dataset.py","file_url":"https://github.com/ubc-vision/LDM_correspondences/blob/HEAD/eval/dataset.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d743428e092c7a2d"}},{"code_sha256_prefix":"988fd31ae49fff76","entry":"get_confirm_token","repo":"ubc-vision/LDM_correspondences","repo_kind":"official","path":"eval/download.py","file_url":"https://github.com/ubc-vision/LDM_correspondences/blob/HEAD/eval/download.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"988fd31ae49fff76"}},{"code_sha256_prefix":"0e293426913c9cf4","entry":"load_checkpoint","repo":"ubc-vision/LDM_correspondences","repo_kind":"official","path":"utils/utils.py","file_url":"https://github.com/ubc-vision/LDM_correspondences/blob/HEAD/utils/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0e293426913c9cf4"}},{"code_sha256_prefix":"d2ba3f4063f64a9f","entry":"load_dataset","repo":"ubc-vision/LDM_correspondences","repo_kind":"official","path":"eval/download.py","file_url":"https://github.com/ubc-vision/LDM_correspondences/blob/HEAD/eval/download.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d2ba3f4063f64a9f"}},{"code_sha256_prefix":"7e8f8e2a3ea7f64e","entry":"parse_list","repo":"ubc-vision/LDM_correspondences","repo_kind":"official","path":"utils/utils.py","file_url":"https://github.com/ubc-vision/LDM_correspondences/blob/HEAD/utils/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7e8f8e2a3ea7f64e"}},{"code_sha256_prefix":"d7fccb2844458579","entry":"random_crop","repo":"ubc-vision/LDM_correspondences","repo_kind":"official","path":"eval/dataset.py","file_url":"https://github.com/ubc-vision/LDM_correspondences/blob/HEAD/eval/dataset.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d7fccb2844458579"}},{"code_sha256_prefix":"3c8be19016fde613","entry":"resize","repo":"ubc-vision/LDM_correspondences","repo_kind":"official","path":"eval/dataset.py","file_url":"https://github.com/ubc-vision/LDM_correspondences/blob/HEAD/eval/dataset.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3c8be19016fde613"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}