{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/splatformer-point-transformer-for-robust-3d","title":"SplatFormer: Point Transformer for Robust 3D Gaussian Splatting","arxiv_id":"2411.06390","date":"2024-11-10","proceeding":null,"authors":["Yutong Chen","Marko Mihajlovic","Xiyi Chen","Yiming Wang","Sergey Prokudin","Siyu Tang"],"abstract":"3D Gaussian Splatting (3DGS) has recently transformed photorealistic reconstruction, achieving high visual fidelity and real-time performance. However, rendering quality significantly deteriorates when test views deviate from the camera angles used during training, posing a major challenge for applications in immersive free-viewpoint rendering and navigation. In this work, we conduct a comprehensive evaluation of 3DGS and related novel view synthesis methods under out-of-distribution (OOD) test camera scenarios. By creating diverse test cases with synthetic and real-world datasets, we demonstrate that most existing methods, including those incorporating various regularization techniques and data-driven priors, struggle to generalize effectively to OOD views. To address this limitation, we introduce SplatFormer, the first point transformer model specifically designed to operate on Gaussian splats. SplatFormer takes as input an initial 3DGS set optimized under limited training views and refines it in a single forward pass, effectively removing potential artifacts in OOD test views. To our knowledge, this is the first successful application of point transformers directly on 3DGS sets, surpassing the limitations of previous multi-scene training methods, which could handle only a restricted number of input views during inference. Our model significantly improves rendering quality under extreme novel views, achieving state-of-the-art performance in these challenging scenarios and outperforming various 3DGS regularization techniques, multi-scene models tailored for sparse view synthesis, and diffusion-based frameworks.","url_abs":"https://arxiv.org/abs/2411.06390v2","url_pdf":"https://arxiv.org/pdf/2411.06390v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"splatformer-point-transformer-for-robust-3d","repo_url":"https://github.com/ChenYutongTHU/SplatFormer","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"3dgs","task_name":"3DGS"},{"task_slug":"novel-view-synthesis","task_name":"Novel View Synthesis"}],"methods":[{"method_slug":"set","method_name":"SET"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2411.06390","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2411.06390"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ChenYutongTHU/SplatFormer","reach":{"status":"ok"}}],"summary":{"ran_honours":3,"ran_draft_wrong":1,"unverified":5},"by_repo_kind":{"official":{"samples":9,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":9,"samples":[{"code_sha256_prefix":"5a8b1e0eb5ec2789","entry":"RGB2SH","repo":"ChenYutongTHU/SplatFormer","repo_kind":"official","path":"utils/gs_utils.py","file_url":"https://github.com/ChenYutongTHU/SplatFormer/blob/HEAD/utils/gs_utils.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5a8b1e0eb5ec2789"}},{"code_sha256_prefix":"c56b7ef16f309a45","entry":"gaussian","repo":"ChenYutongTHU/SplatFormer","repo_kind":"official","path":"utils/metrics.py","file_url":"https://github.com/ChenYutongTHU/SplatFormer/blob/HEAD/utils/metrics.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c56b7ef16f309a45"}},{"code_sha256_prefix":"8a38e306ff4c5b42","entry":"read_cameras_text","repo":"ChenYutongTHU/SplatFormer","repo_kind":"official","path":"dataset/colmap_utils.py","file_url":"https://github.com/ChenYutongTHU/SplatFormer/blob/HEAD/dataset/colmap_utils.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8a38e306ff4c5b42"}},{"code_sha256_prefix":"56858e04e6fdb2ff","entry":"read_next_bytes","repo":"ChenYutongTHU/SplatFormer","repo_kind":"official","path":"dataset/colmap_utils.py","file_url":"https://github.com/ChenYutongTHU/SplatFormer/blob/HEAD/dataset/colmap_utils.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"56858e04e6fdb2ff"}},{"code_sha256_prefix":"55e5c8b4a3e6a864","entry":"SH2RGB","repo":"ChenYutongTHU/SplatFormer","repo_kind":"official","path":"utils/gs_utils.py","file_url":"https://github.com/ChenYutongTHU/SplatFormer/blob/HEAD/utils/gs_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"55e5c8b4a3e6a864"}},{"code_sha256_prefix":"597b0fba263e2c4c","entry":"move_to_device","repo":"ChenYutongTHU/SplatFormer","repo_kind":"official","path":"utils/gpu_utils.py","file_url":"https://github.com/ChenYutongTHU/SplatFormer/blob/HEAD/utils/gpu_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"597b0fba263e2c4c"}},{"code_sha256_prefix":"6e14e874e7f40344","entry":"mse","repo":"ChenYutongTHU/SplatFormer","repo_kind":"official","path":"utils/metrics.py","file_url":"https://github.com/ChenYutongTHU/SplatFormer/blob/HEAD/utils/metrics.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6e14e874e7f40344"}},{"code_sha256_prefix":"a52be3d2bf850199","entry":"psnr","repo":"ChenYutongTHU/SplatFormer","repo_kind":"official","path":"utils/metrics.py","file_url":"https://github.com/ChenYutongTHU/SplatFormer/blob/HEAD/utils/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a52be3d2bf850199"}},{"code_sha256_prefix":"4a20cdd355703af3","entry":"read_cameras_binary","repo":"ChenYutongTHU/SplatFormer","repo_kind":"official","path":"dataset/colmap_utils.py","file_url":"https://github.com/ChenYutongTHU/SplatFormer/blob/HEAD/dataset/colmap_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4a20cdd355703af3"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}