{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/inst-inpaint-instructing-to-remove-objects","title":"Inst-Inpaint: Instructing to Remove Objects with Diffusion Models","arxiv_id":"2304.03246","date":"2023-04-06","proceeding":null,"authors":["Ahmet Burak Yildirim","Vedat Baday","Erkut Erdem","Aykut Erdem","Aysegul Dundar"],"abstract":"Image inpainting task refers to erasing unwanted pixels from images and filling them in a semantically consistent and realistic way. Traditionally, the pixels that are wished to be erased are defined with binary masks. From the application point of view, a user needs to generate the masks for the objects they would like to remove which can be time-consuming and prone to errors. In this work, we are interested in an image inpainting algorithm that estimates which object to be removed based on natural language input and removes it, simultaneously. For this purpose, first, we construct a dataset named GQA-Inpaint for this task. Second, we present a novel inpainting framework, Inst-Inpaint, that can remove objects from images based on the instructions given as text prompts. We set various GAN and diffusion-based baselines and run experiments on synthetic and real image datasets. We compare methods with different evaluation metrics that measure the quality and accuracy of the models and show significant quantitative and qualitative improvements.","url_abs":"https://arxiv.org/abs/2304.03246v2","url_pdf":"https://arxiv.org/pdf/2304.03246v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"inst-inpaint-instructing-to-remove-objects","repo_url":"https://github.com/abyildirim/inst-inpaint","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"image-inpainting","task_name":"Image Inpainting"}],"methods":[{"method_slug":"pixel-prediction","method_name":"Inpainting"},{"method_slug":"latent-diffusion-model","method_name":"Latent Diffusion Model"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2304.03246","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2304.03246"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/abyildirim/inst-inpaint","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":2,"ran_violates":2,"ran":2,"unverified":3},"by_repo_kind":{"official":{"samples":9,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"fe5dd5258046898c","entry":"always","repo":"abyildirim/inst-inpaint","repo_kind":"official","path":"ldm/modules/x_transformer.py","file_url":"https://github.com/abyildirim/inst-inpaint/blob/HEAD/ldm/modules/x_transformer.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fe5dd5258046898c"}},{"code_sha256_prefix":"424012cb37b31172","entry":"default","repo":"abyildirim/inst-inpaint","repo_kind":"official","path":"ldm/modules/attention.py","file_url":"https://github.com/abyildirim/inst-inpaint/blob/HEAD/ldm/modules/attention.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"424012cb37b31172"}},{"code_sha256_prefix":"aa5486a3650902d8","entry":"exists","repo":"abyildirim/inst-inpaint","repo_kind":"official","path":"ldm/modules/attention.py","file_url":"https://github.com/abyildirim/inst-inpaint/blob/HEAD/ldm/modules/attention.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"aa5486a3650902d8"}},{"code_sha256_prefix":"b1368330cf0f5642","entry":"isimage","repo":"abyildirim/inst-inpaint","repo_kind":"official","path":"ldm/util.py","file_url":"https://github.com/abyildirim/inst-inpaint/blob/HEAD/ldm/util.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b1368330cf0f5642"}},{"code_sha256_prefix":"d72762b700feee6f","entry":"ismap","repo":"abyildirim/inst-inpaint","repo_kind":"official","path":"ldm/util.py","file_url":"https://github.com/abyildirim/inst-inpaint/blob/HEAD/ldm/util.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d72762b700feee6f"}},{"code_sha256_prefix":"9a299fe5ae09e407","entry":"uniq","repo":"abyildirim/inst-inpaint","repo_kind":"official","path":"ldm/modules/attention.py","file_url":"https://github.com/abyildirim/inst-inpaint/blob/HEAD/ldm/modules/attention.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9a299fe5ae09e407"}},{"code_sha256_prefix":"f0cd563bbffad24a","entry":"get_frechet_inception_distance","repo":"abyildirim/inst-inpaint","repo_kind":"official","path":"eval/fid.py","file_url":"https://github.com/abyildirim/inst-inpaint/blob/HEAD/eval/fid.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f0cd563bbffad24a"}},{"code_sha256_prefix":"f9bd2e83191afad1","entry":"log_txt_as_img","repo":"abyildirim/inst-inpaint","repo_kind":"official","path":"ldm/util.py","file_url":"https://github.com/abyildirim/inst-inpaint/blob/HEAD/ldm/util.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f9bd2e83191afad1"}},{"code_sha256_prefix":"db96c313cd7cb34d","entry":"preprocess_image","repo":"abyildirim/inst-inpaint","repo_kind":"official","path":"gradio_app/utils.py","file_url":"https://github.com/abyildirim/inst-inpaint/blob/HEAD/gradio_app/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"db96c313cd7cb34d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}