{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/matfuse-controllable-material-generation-with","title":"MatFuse: Controllable Material Generation with Diffusion Models","arxiv_id":"2308.11408","date":"2023-08-22","proceeding":"CVPR 2024 1","authors":["Giuseppe Vecchio","Renato Sortino","Simone Palazzo","Concetto Spampinato"],"abstract":"Creating high-quality materials in computer graphics is a challenging and time-consuming task, which requires great expertise. To simplify this process, we introduce MatFuse, a unified approach that harnesses the generative power of diffusion models for creation and editing of 3D materials. Our method integrates multiple sources of conditioning, including color palettes, sketches, text, and pictures, enhancing creative possibilities and granting fine-grained control over material synthesis. Additionally, MatFuse enables map-level material editing capabilities through latent manipulation by means of a multi-encoder compression model which learns a disentangled latent representation for each map. We demonstrate the effectiveness of MatFuse under multiple conditioning settings and explore the potential of material editing. Finally, we assess the quality of the generated materials both quantitatively in terms of CLIP-IQA and FID scores and qualitatively by conducting a user study. Source code for training MatFuse and supplemental materials are publicly available at https://gvecchio.com/matfuse.","url_abs":"https://arxiv.org/abs/2308.11408v3","url_pdf":"https://arxiv.org/pdf/2308.11408v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"matfuse-controllable-material-generation-with","repo_url":"https://github.com/giuvecchio/matfuse-sd","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"svbrdf-estimation","task_name":"SVBRDF Estimation"}],"methods":[{"method_slug":"diffusion","method_name":"Diffusion"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2308.11408","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2308.11408"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/giuvecchio/matfuse-sd","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":5,"ran_draft_wrong":2,"ran_violates":2,"unverified":3},"by_repo_kind":{"official":{"samples":12,"ran":9,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"4950c5f467f89676","entry":"Normalize","repo":"giuvecchio/matfuse-sd","repo_kind":"official","path":"src/diffusers_pipeline/vae_matfuse.py","file_url":"https://github.com/giuvecchio/matfuse-sd/blob/HEAD/src/diffusers_pipeline/vae_matfuse.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4950c5f467f89676"}},{"code_sha256_prefix":"fe5dd5258046898c","entry":"always","repo":"giuvecchio/matfuse-sd","repo_kind":"official","path":"src/ldm/modules/x_transformer.py","file_url":"https://github.com/giuvecchio/matfuse-sd/blob/HEAD/src/ldm/modules/x_transformer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fe5dd5258046898c"}},{"code_sha256_prefix":"424012cb37b31172","entry":"default","repo":"giuvecchio/matfuse-sd","repo_kind":"official","path":"src/ldm/modules/attention.py","file_url":"https://github.com/giuvecchio/matfuse-sd/blob/HEAD/src/ldm/modules/attention.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"424012cb37b31172"}},{"code_sha256_prefix":"aa5486a3650902d8","entry":"exists","repo":"giuvecchio/matfuse-sd","repo_kind":"official","path":"src/ldm/modules/attention.py","file_url":"https://github.com/giuvecchio/matfuse-sd/blob/HEAD/src/ldm/modules/attention.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"aa5486a3650902d8"}},{"code_sha256_prefix":"d72762b700feee6f","entry":"ismap","repo":"giuvecchio/matfuse-sd","repo_kind":"official","path":"src/ldm/util.py","file_url":"https://github.com/giuvecchio/matfuse-sd/blob/HEAD/src/ldm/util.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d72762b700feee6f"}},{"code_sha256_prefix":"328833bf6da7c55a","entry":"nonlinearity","repo":"giuvecchio/matfuse-sd","repo_kind":"official","path":"src/diffusers_pipeline/vae_matfuse.py","file_url":"https://github.com/giuvecchio/matfuse-sd/blob/HEAD/src/diffusers_pipeline/vae_matfuse.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"328833bf6da7c55a"}},{"code_sha256_prefix":"adec46d52e489fb1","entry":"process_palette","repo":"giuvecchio/matfuse-sd","repo_kind":"official","path":"src/utils/inference_helpers.py","file_url":"https://github.com/giuvecchio/matfuse-sd/blob/HEAD/src/utils/inference_helpers.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"adec46d52e489fb1"}},{"code_sha256_prefix":"38d99fca819d49e9","entry":"process_sketch","repo":"giuvecchio/matfuse-sd","repo_kind":"official","path":"src/utils/inference_helpers.py","file_url":"https://github.com/giuvecchio/matfuse-sd/blob/HEAD/src/utils/inference_helpers.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"38d99fca819d49e9"}},{"code_sha256_prefix":"9a299fe5ae09e407","entry":"uniq","repo":"giuvecchio/matfuse-sd","repo_kind":"official","path":"src/ldm/modules/attention.py","file_url":"https://github.com/giuvecchio/matfuse-sd/blob/HEAD/src/ldm/modules/attention.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9a299fe5ae09e407"}},{"code_sha256_prefix":"357da20145f46520","entry":"log_txt_as_img","repo":"giuvecchio/matfuse-sd","repo_kind":"official","path":"src/ldm/util.py","file_url":"https://github.com/giuvecchio/matfuse-sd/blob/HEAD/src/ldm/util.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"357da20145f46520"}},{"code_sha256_prefix":"57dc047ce3d16a0b","entry":"process_image","repo":"giuvecchio/matfuse-sd","repo_kind":"official","path":"src/utils/inference_helpers.py","file_url":"https://github.com/giuvecchio/matfuse-sd/blob/HEAD/src/utils/inference_helpers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"57dc047ce3d16a0b"}},{"code_sha256_prefix":"0c856c059e6cecce","entry":"visualize_palette","repo":"giuvecchio/matfuse-sd","repo_kind":"official","path":"src/ldm/util.py","file_url":"https://github.com/giuvecchio/matfuse-sd/blob/HEAD/src/ldm/util.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0c856c059e6cecce"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}