{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/equiformerv2-improved-equivariant-transformer","title":"EquiformerV2: Improved Equivariant Transformer for Scaling to Higher-Degree Representations","arxiv_id":"2306.12059","date":"2023-06-21","proceeding":null,"authors":["Yi-Lun Liao","Brandon Wood","Abhishek Das","Tess Smidt"],"abstract":"Equivariant Transformers such as Equiformer have demonstrated the efficacy of applying Transformers to the domain of 3D atomistic systems. However, they are limited to small degrees of equivariant representations due to their computational complexity. In this paper, we investigate whether these architectures can scale well to higher degrees. Starting from Equiformer, we first replace $SO(3)$ convolutions with eSCN convolutions to efficiently incorporate higher-degree tensors. Then, to better leverage the power of higher degrees, we propose three architectural improvements -- attention re-normalization, separable $S^2$ activation and separable layer normalization. Putting this all together, we propose EquiformerV2, which outperforms previous state-of-the-art methods on large-scale OC20 dataset by up to $9\\%$ on forces, $4\\%$ on energies, offers better speed-accuracy trade-offs, and $2\\times$ reduction in DFT calculations needed for computing adsorption energies. Additionally, EquiformerV2 trained on only OC22 dataset outperforms GemNet-OC trained on both OC20 and OC22 datasets, achieving much better data efficiency. Finally, we compare EquiformerV2 with Equiformer on QM9 and OC20 S2EF-2M datasets to better understand the performance gain brought by higher degrees.","url_abs":"https://arxiv.org/abs/2306.12059v3","url_pdf":"https://arxiv.org/pdf/2306.12059v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"equiformerv2-improved-equivariant-transformer","repo_url":"https://github.com/atomicarchitects/equiformer_v2","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"equiformerv2-improved-equivariant-transformer","repo_url":"https://github.com/atomicarchitects/dens","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"graph-property-prediction","task_name":"Graph Property Prediction"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/graph-property-prediction-on-qm9","task":"Graph Property Prediction","dataset":"QM9","model":"EquiformerV2","rank_in_archive_order":4,"of":10,"metrics":{"Standardized MAE":"0.67","alpha (ma)":"47","gap (meV)":"29","logMAE":"-5.87"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2306.12059","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2306.12059"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/atomicarchitects/equiformer_v2","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/atomicarchitects/dens","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_fixture":1,"ran_violates":1,"ran_honours":1,"unverified":2},"by_repo_kind":{"official":{"samples":5,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"3ac6b7d76e8e3584","entry":"drop_path","repo":"atomicarchitects/equiformer_v2","repo_kind":"official","path":"nets/equiformer_v2/drop.py","file_url":"https://github.com/atomicarchitects/equiformer_v2/blob/HEAD/nets/equiformer_v2/drop.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3ac6b7d76e8e3584"}},{"code_sha256_prefix":"294ba20b052bfbc7","entry":"gaussian","repo":"atomicarchitects/equiformer_v2","repo_kind":"official","path":"nets/equiformer_v2/gaussian_rbf.py","file_url":"https://github.com/atomicarchitects/equiformer_v2/blob/HEAD/nets/equiformer_v2/gaussian_rbf.py","link_basis":"plan_row","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"294ba20b052bfbc7"}},{"code_sha256_prefix":"d4c1ba3933fbb7ac","entry":"get_l_to_all_m_expand_index","repo":"atomicarchitects/equiformer_v2","repo_kind":"official","path":"nets/equiformer_v2/layer_norm.py","file_url":"https://github.com/atomicarchitects/equiformer_v2/blob/HEAD/nets/equiformer_v2/layer_norm.py","link_basis":"plan_row","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d4c1ba3933fbb7ac"}},{"code_sha256_prefix":"f58e462fb9c75bb8","entry":"get_normalization_layer","repo":"atomicarchitects/equiformer_v2","repo_kind":"official","path":"nets/equiformer_v2/layer_norm.py","file_url":"https://github.com/atomicarchitects/equiformer_v2/blob/HEAD/nets/equiformer_v2/layer_norm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f58e462fb9c75bb8"}},{"code_sha256_prefix":"b316bf7a13f27e97","entry":"init_edge_rot_mat","repo":"atomicarchitects/equiformer_v2","repo_kind":"official","path":"nets/equiformer_v2/edge_rot_mat.py","file_url":"https://github.com/atomicarchitects/equiformer_v2/blob/HEAD/nets/equiformer_v2/edge_rot_mat.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b316bf7a13f27e97"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}