{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/dpot-auto-regressive-denoising-operator","title":"DPOT: Auto-Regressive Denoising Operator Transformer for Large-Scale PDE Pre-Training","arxiv_id":"2403.03542","date":"2024-03-06","proceeding":null,"authors":["Zhongkai Hao","Chang Su","Songming Liu","Julius Berner","Chengyang Ying","Hang Su","Anima Anandkumar","Jian Song","Jun Zhu"],"abstract":"Pre-training has been investigated to improve the efficiency and performance of training neural operators in data-scarce settings. However, it is largely in its infancy due to the inherent complexity and diversity, such as long trajectories, multiple scales and varying dimensions of partial differential equations (PDEs) data. In this paper, we present a new auto-regressive denoising pre-training strategy, which allows for more stable and efficient pre-training on PDE data and generalizes to various downstream tasks. Moreover, by designing a flexible and scalable model architecture based on Fourier attention, we can easily scale up the model for large-scale pre-training. We train our PDE foundation model with up to 0.5B parameters on 10+ PDE datasets with more than 100k trajectories. Extensive experiments show that we achieve SOTA on these benchmarks and validate the strong generalizability of our model to significantly enhance performance on diverse downstream PDE tasks like 3D data. Code is available at \\url{https://github.com/thu-ml/DPOT}.","url_abs":"https://arxiv.org/abs/2403.03542v4","url_pdf":"https://arxiv.org/pdf/2403.03542v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"dpot-auto-regressive-denoising-operator","repo_url":"https://github.com/HaoZhongkai/DPOT","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"dpot-auto-regressive-denoising-operator","repo_url":"https://github.com/thu-ml/dpot","is_official":0,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"denoising","task_name":"Denoising"},{"task_slug":"diversity","task_name":"Diversity"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2403.03542","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.03542"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/thu-ml/dpot","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/HaoZhongkai/DPOT","reach":null}],"summary":{"ran":3,"ran_draft_wrong":1,"ran_fixture":1,"unverified":2},"by_repo_kind":{"official":{"samples":5,"ran":3,"repositories":1},"named_in_paper":{"samples":2,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":7,"samples":[{"code_sha256_prefix":"7f3008c73bcacbe1","entry":"AFNO2D","repo":"HaoZhongkai/DPOT","repo_kind":"official","path":"models/dpot.py","file_url":"https://github.com/HaoZhongkai/DPOT/blob/HEAD/models/dpot.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7f3008c73bcacbe1"}},{"code_sha256_prefix":"994b3f1e743c60f0","entry":"Block","repo":"HaoZhongkai/DPOT","repo_kind":"official","path":"models/dpot.py","file_url":"https://github.com/HaoZhongkai/DPOT/blob/HEAD/models/dpot.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"994b3f1e743c60f0"}},{"code_sha256_prefix":"89079a92fee2d0b3","entry":"PatchEmbed","repo":"HaoZhongkai/DPOT","repo_kind":"official","path":"models/dpot.py","file_url":"https://github.com/HaoZhongkai/DPOT/blob/HEAD/models/dpot.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"89079a92fee2d0b3"}},{"code_sha256_prefix":"00a42186a37558bd","entry":"checkpoint_filter_fn","repo":"thu-ml/dpot","repo_kind":"named_in_paper","path":"models/dpot.py","file_url":"https://github.com/thu-ml/dpot/blob/HEAD/models/dpot.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"00a42186a37558bd"}},{"code_sha256_prefix":"c79cae123eef4714","entry":"resize_pos_embed","repo":"thu-ml/dpot","repo_kind":"named_in_paper","path":"models/dpot.py","file_url":"https://github.com/thu-ml/dpot/blob/HEAD/models/dpot.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c79cae123eef4714"}},{"code_sha256_prefix":"87ce4cc322af7a18","entry":"DPOTNet","repo":"HaoZhongkai/DPOT","repo_kind":"official","path":"models/dpot.py","file_url":"https://github.com/HaoZhongkai/DPOT/blob/HEAD/models/dpot.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"87ce4cc322af7a18"}},{"code_sha256_prefix":"13465fb7dee87bc7","entry":"TimeAggregator","repo":"HaoZhongkai/DPOT","repo_kind":"official","path":"models/dpot.py","file_url":"https://github.com/HaoZhongkai/DPOT/blob/HEAD/models/dpot.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"13465fb7dee87bc7"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}