{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/sorbet-a-neuromorphic-hardware-compatible","title":"Sorbet: A Neuromorphic Hardware-Compatible Transformer-Based Spiking Language Model","arxiv_id":"2409.15298","date":"2024-09-04","proceeding":null,"authors":["Kaiwen Tang","Zhanglu Yan","Weng-Fai Wong"],"abstract":"For reasons such as privacy, there are use cases for language models at the edge. This has given rise to small language models (SLMs) targeted for deployment in resource-constrained devices where energy efficiency is a significant concern. Spiking neural networks (SNNs) offer a promising solution due to their energy efficiency, and there are already works on realizing transformer-based models on SNNs. However, key operations like softmax and layer normalization (LN) are difficult to implement on neuromorphic hardware, and many of these early works sidestepped them. To address these challenges, we introduce Sorbet, a transformer-based spiking language model that is more neuromorphic hardware-compatible. Sorbet incorporates a novel shifting-based softmax called PTsoftmax and a power normalization method using bit-shifting (BSPN), both designed to replace the respective energy-intensive operations. By leveraging knowledge distillation and model quantization, Sorbet achieved a highly compressed binary weight model that maintains competitive performance while significantly reducing energy consumption. We validate Sorbet's effectiveness through extensive testing on the GLUE benchmark and a series of ablation studies, demonstrating its potential as an energy-efficient solution for language model inference.","url_abs":"https://arxiv.org/abs/2409.15298v1","url_pdf":"https://arxiv.org/pdf/2409.15298v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[],"tasks":[{"task_slug":"knowledge-distillation","task_name":"Knowledge Distillation"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"quantization","task_name":"Quantization"}],"methods":[{"method_slug":"knowledge-distillation","method_name":"Knowledge Distillation"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"softmax","method_name":"Softmax"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2409.15298","atlas_url":"https://app.syntology.ai/?focus=2409.15298","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2409.15298"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/Kaiwen-Tang/Sorbet","reach":null}],"summary":{"ran":3,"ran_honours":1},"by_repo_kind":{"found_in_text":{"samples":4,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":4,"samples":[{"code_sha256_prefix":"3be9f44e0c56d62d","entry":"GroupScaling1D","repo":"Kaiwen-Tang/Sorbet","repo_kind":"found_in_text","path":"transformer/norm_new.py","file_url":"https://github.com/Kaiwen-Tang/Sorbet/blob/HEAD/transformer/norm_new.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3be9f44e0c56d62d"}},{"code_sha256_prefix":"204657f6f929936b","entry":"MaskPowerNorm","repo":"Kaiwen-Tang/Sorbet","repo_kind":"found_in_text","path":"transformer/norm_new.py","file_url":"https://github.com/Kaiwen-Tang/Sorbet/blob/HEAD/transformer/norm_new.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"204657f6f929936b"}},{"code_sha256_prefix":"2591733bf8fe99bc","entry":"PowerFunction","repo":"Kaiwen-Tang/Sorbet","repo_kind":"found_in_text","path":"transformer/norm_new.py","file_url":"https://github.com/Kaiwen-Tang/Sorbet/blob/HEAD/transformer/norm_new.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2591733bf8fe99bc"}},{"code_sha256_prefix":"b33fa9d5df8b0e4f","entry":"find_closest_power_of_two","repo":"Kaiwen-Tang/Sorbet","repo_kind":"found_in_text","path":"transformer/norm_new.py","file_url":"https://github.com/Kaiwen-Tang/Sorbet/blob/HEAD/transformer/norm_new.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b33fa9d5df8b0e4f"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}