{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/gradient-driven-3d-segmentation-and","title":"Gradient-Driven 3D Segmentation and Affordance Transfer in Gaussian Splatting Using 2D Masks","arxiv_id":"2409.11681","date":"2024-09-18","proceeding":null,"authors":["Joji Joseph","Bharadwaj Amrutur","Shalabh Bhatnagar"],"abstract":"3D Gaussian Splatting has emerged as a powerful 3D scene representation technique, capturing fine details with high efficiency. In this paper, we introduce a novel voting-based method that extends 2D segmentation models to 3D Gaussian splats. Our approach leverages masked gradients, where gradients are filtered by input 2D masks, and these gradients are used as votes to achieve accurate segmentation. As a byproduct, we discovered that inference-time gradients can also be used to prune Gaussians, resulting in up to 21% compression. Additionally, we explore few-shot affordance transfer, allowing annotations from 2D images to be effectively transferred onto 3D Gaussian splats. The robust yet straightforward mathematical formulation underlying this approach makes it a highly effective tool for numerous downstream applications, such as augmented reality (AR), object editing, and robotics. The project code and additional resources are available at https://jojijoseph.github.io/3dgs-segmentation.","url_abs":"https://arxiv.org/abs/2409.11681v1","url_pdf":"https://arxiv.org/pdf/2409.11681v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"gradient-driven-3d-segmentation-and","repo_url":"https://github.com/JojiJoseph/3dgs-gradient-segmentation","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"3dgs","task_name":"3DGS"},{"task_slug":"segmentation","task_name":"Segmentation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2409.11681","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2409.11681"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/JojiJoseph/3dgs-gradient-segmentation","reach":null}],"summary":{"ran_honours":3,"ran_draft_wrong":1},"by_repo_kind":{"official":{"samples":4,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":4,"samples":[{"code_sha256_prefix":"0369e0f374f92d54","entry":"create_checkerboard","repo":"JojiJoseph/3dgs-gradient-segmentation","repo_kind":"official","path":"viewer.py","file_url":"https://github.com/JojiJoseph/3dgs-gradient-segmentation/blob/HEAD/viewer.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"0369e0f374f92d54"}},{"code_sha256_prefix":"fe60da60fcc93260","entry":"get_rpy_matrix","repo":"JojiJoseph/3dgs-gradient-segmentation","repo_kind":"official","path":"viewer.py","file_url":"https://github.com/JojiJoseph/3dgs-gradient-segmentation/blob/HEAD/viewer.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"fe60da60fcc93260"}},{"code_sha256_prefix":"61ea259b82c0f00e","entry":"get_viewmat_from_colmap_image","repo":"JojiJoseph/3dgs-gradient-segmentation","repo_kind":"official","path":"affordance_transfer_pipeline.py","file_url":"https://github.com/JojiJoseph/3dgs-gradient-segmentation/blob/HEAD/affordance_transfer_pipeline.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"61ea259b82c0f00e"}},{"code_sha256_prefix":"c79bd872057a2f93","entry":"torch_to_cv","repo":"JojiJoseph/3dgs-gradient-segmentation","repo_kind":"official","path":"affordance_transfer_pipeline.py","file_url":"https://github.com/JojiJoseph/3dgs-gradient-segmentation/blob/HEAD/affordance_transfer_pipeline.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"c79bd872057a2f93"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}