{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/beyond-surface-statistics-scene","title":"Beyond Surface Statistics: Scene Representations in a Latent Diffusion Model","arxiv_id":"2306.05720","date":"2023-06-09","proceeding":null,"authors":["Yida Chen","Fernanda Viégas","Martin Wattenberg"],"abstract":"Latent diffusion models (LDMs) exhibit an impressive ability to produce realistic images, yet the inner workings of these models remain mysterious. Even when trained purely on images without explicit depth information, they typically output coherent pictures of 3D scenes. In this work, we investigate a basic interpretability question: does an LDM create and use an internal representation of simple scene geometry? Using linear probes, we find evidence that the internal activations of the LDM encode linear representations of both 3D depth data and a salient-object / background distinction. These representations appear surprisingly early in the denoising process$-$well before a human can easily make sense of the noisy images. Intervention experiments further indicate these representations play a causal role in image synthesis, and may be used for simple high-level editing of an LDM's output. Project page: https://yc015.github.io/scene-representation-diffusion-model/","url_abs":"https://arxiv.org/abs/2306.05720v2","url_pdf":"https://arxiv.org/pdf/2306.05720v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"beyond-surface-statistics-scene","repo_url":"https://github.com/yc015/scene-representation-diffusion-model","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"denoising","task_name":"Denoising"},{"task_slug":"image-generation","task_name":"Image Generation"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2306.05720","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.05720"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yc015/scene-representation-diffusion-model","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":4,"unverified":10},"by_repo_kind":{"official":{"samples":14,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"25ba9b5031480973","entry":"compute_scale_and_shift","repo":"yc015/scene-representation-diffusion-model","repo_kind":"official","path":"probe_src/depth_loss.py","file_url":"https://github.com/yc015/scene-representation-diffusion-model/blob/HEAD/probe_src/depth_loss.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"25ba9b5031480973"}},{"code_sha256_prefix":"eafa252fd6f86857","entry":"get_parameter_device","repo":"yc015/scene-representation-diffusion-model","repo_kind":"official","path":"modified_diffusers/modeling_utils.py","file_url":"https://github.com/yc015/scene-representation-diffusion-model/blob/HEAD/modified_diffusers/modeling_utils.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"eafa252fd6f86857"}},{"code_sha256_prefix":"dc2045840ce679cb","entry":"get_parameter_dtype","repo":"yc015/scene-representation-diffusion-model","repo_kind":"official","path":"modified_diffusers/modeling_utils.py","file_url":"https://github.com/yc015/scene-representation-diffusion-model/blob/HEAD/modified_diffusers/modeling_utils.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dc2045840ce679cb"}},{"code_sha256_prefix":"e8996630fe3501e1","entry":"reduction_batch_based","repo":"yc015/scene-representation-diffusion-model","repo_kind":"official","path":"probe_src/depth_loss.py","file_url":"https://github.com/yc015/scene-representation-diffusion-model/blob/HEAD/probe_src/depth_loss.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e8996630fe3501e1"}},{"code_sha256_prefix":"301682fc3342590a","entry":"check_imports","repo":"yc015/scene-representation-diffusion-model","repo_kind":"official","path":"modified_diffusers/dynamic_modules_utils.py","file_url":"https://github.com/yc015/scene-representation-diffusion-model/blob/HEAD/modified_diffusers/dynamic_modules_utils.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"301682fc3342590a"}},{"code_sha256_prefix":"a580fad941fd2f88","entry":"cosine_distance","repo":"yc015/scene-representation-diffusion-model","repo_kind":"official","path":"modified_diffusers/pipelines/stable_diffusion/safety_checker.py","file_url":"https://github.com/yc015/scene-representation-diffusion-model/blob/HEAD/modified_diffusers/pipelines/stable_diffusion/safety_checker.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a580fad941fd2f88"}},{"code_sha256_prefix":"00cc853a73f3aeb6","entry":"get_relative_import_files","repo":"yc015/scene-representation-diffusion-model","repo_kind":"official","path":"modified_diffusers/dynamic_modules_utils.py","file_url":"https://github.com/yc015/scene-representation-diffusion-model/blob/HEAD/modified_diffusers/dynamic_modules_utils.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"00cc853a73f3aeb6"}},{"code_sha256_prefix":"450aeffe7f80f917","entry":"get_relative_imports","repo":"yc015/scene-representation-diffusion-model","repo_kind":"official","path":"modified_diffusers/dynamic_modules_utils.py","file_url":"https://github.com/yc015/scene-representation-diffusion-model/blob/HEAD/modified_diffusers/dynamic_modules_utils.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"450aeffe7f80f917"}},{"code_sha256_prefix":"77bd5e6e8ca8c91d","entry":"load_state_dict","repo":"yc015/scene-representation-diffusion-model","repo_kind":"official","path":"modified_diffusers/modeling_utils.py","file_url":"https://github.com/yc015/scene-representation-diffusion-model/blob/HEAD/modified_diffusers/modeling_utils.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"77bd5e6e8ca8c91d"}},{"code_sha256_prefix":"1090b7fe2da4f983","entry":"make_counterfactual_label","repo":"yc015/scene-representation-diffusion-model","repo_kind":"official","path":"probe_src/continuous_depth_intervention_utils.py","file_url":"https://github.com/yc015/scene-representation-diffusion-model/blob/HEAD/probe_src/continuous_depth_intervention_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1090b7fe2da4f983"}},{"code_sha256_prefix":"002691469b74a353","entry":"make_counterfactual_label","repo":"yc015/scene-representation-diffusion-model","repo_kind":"official","path":"probe_src/depth_intervention_utils.py","file_url":"https://github.com/yc015/scene-representation-diffusion-model/blob/HEAD/probe_src/depth_intervention_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"002691469b74a353"}},{"code_sha256_prefix":"16f5f8b292ace798","entry":"optimize_one_inter_rep","repo":"yc015/scene-representation-diffusion-model","repo_kind":"official","path":"probe_src/continuous_depth_intervention_utils.py","file_url":"https://github.com/yc015/scene-representation-diffusion-model/blob/HEAD/probe_src/continuous_depth_intervention_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"16f5f8b292ace798"}},{"code_sha256_prefix":"b31fe2f42ad6b09d","entry":"optimize_one_inter_rep","repo":"yc015/scene-representation-diffusion-model","repo_kind":"official","path":"probe_src/depth_intervention_utils.py","file_url":"https://github.com/yc015/scene-representation-diffusion-model/blob/HEAD/probe_src/depth_intervention_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b31fe2f42ad6b09d"}},{"code_sha256_prefix":"c92f0115a72bf454","entry":"reduction_image_based","repo":"yc015/scene-representation-diffusion-model","repo_kind":"official","path":"probe_src/depth_loss.py","file_url":"https://github.com/yc015/scene-representation-diffusion-model/blob/HEAD/probe_src/depth_loss.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c92f0115a72bf454"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}