{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/scattered-mixture-of-experts-implementation","title":"Scattered Mixture-of-Experts Implementation","arxiv_id":"2403.08245","date":"2024-03-13","proceeding":null,"authors":["Shawn Tan","Yikang Shen","Rameswar Panda","Aaron Courville"],"abstract":"We present ScatterMoE, an implementation of Sparse Mixture-of-Experts (SMoE) on GPUs. ScatterMoE builds upon existing implementations, and overcoming some of the limitations to improve inference and training speed, and memory footprint. This implementation achieves this by avoiding padding and making excessive copies of the input. We introduce ParallelLinear, the main component we use to build our implementation and the various kernels used to speed up the operation. We benchmark our implementation against Megablocks, and show that it enables a higher throughput and lower memory footprint. We also show how ParallelLinear enables extension of the Mixture-of-Experts concept by demonstrating with an implementation of Mixture of Attention.","url_abs":"https://arxiv.org/abs/2403.08245v2","url_pdf":"https://arxiv.org/pdf/2403.08245v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"scattered-mixture-of-experts-implementation","repo_url":"https://github.com/shawntan/scattermoe","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"scattered-mixture-of-experts-implementation","repo_url":"https://github.com/ibm/dolomite-engine","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"scattered-mixture-of-experts-implementation","repo_url":"https://github.com/shawntan/stickbreaking-attention","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"mixture-of-experts","task_name":"Mixture-of-Experts"}],"methods":[{"method_slug":"speed","method_name":"SPEED"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2403.08245","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.08245"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/shawntan/stickbreaking-attention","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ibm/dolomite-engine","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/shawntan/scattermoe","reach":null}],"summary":{"ran_fixture":1,"unverified":2},"by_repo_kind":{"official":{"samples":3,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"89124f53c7727838","entry":"compileable_bincount","repo":"shawntan/scattermoe","repo_kind":"official","path":"scattermoe/parallel_experts.py","file_url":"https://github.com/shawntan/scattermoe/blob/HEAD/scattermoe/parallel_experts.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"89124f53c7727838"}},{"code_sha256_prefix":"99e5dd5a85a8d723","entry":"flatten_sort_count","repo":"shawntan/scattermoe","repo_kind":"official","path":"scattermoe/parallel_experts.py","file_url":"https://github.com/shawntan/scattermoe/blob/HEAD/scattermoe/parallel_experts.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"99e5dd5a85a8d723"}},{"code_sha256_prefix":"6f4324a70935be68","entry":"parallel_linear","repo":"shawntan/scattermoe","repo_kind":"official","path":"scattermoe/parallel_experts.py","file_url":"https://github.com/shawntan/scattermoe/blob/HEAD/scattermoe/parallel_experts.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6f4324a70935be68"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}