{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/gated-delta-networks-improving-mamba2-with","title":"Gated Delta Networks: Improving Mamba2 with Delta Rule","arxiv_id":"2412.06464","date":"2024-12-09","proceeding":null,"authors":["Songlin Yang","Jan Kautz","Ali Hatamizadeh"],"abstract":"Linear Transformers have gained attention as efficient alternatives to standard Transformers, but their performance in retrieval and long-context tasks has been limited. To address these limitations, recent work has explored two distinct mechanisms: gating for adaptive memory control and the delta update rule for precise memory modifications. We observe that these mechanisms are complementary: gating enables rapid memory erasure while the delta rule facilitates targeted updates. Building on this insight, we introduce the gated delta rule and develop a parallel training algorithm optimized for modern hardware. Our proposed architecture, Gated DeltaNet, consistently surpasses existing models like Mamba2 and DeltaNet across multiple benchmarks, including language modeling, common-sense reasoning, in-context retrieval, length extrapolation, and long-context understanding. We further enhance performance by developing hybrid architectures that combine Gated DeltaNet layers with sliding window attention or Mamba2 layers, achieving both improved training efficiency and superior task performance.","url_abs":"https://arxiv.org/abs/2412.06464v1","url_pdf":"https://arxiv.org/pdf/2412.06464v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"gated-delta-networks-improving-mamba2-with","repo_url":"https://github.com/NVlabs/GatedDeltaNet","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"gated-delta-networks-improving-mamba2-with","repo_url":"https://github.com/fla-org/flash-linear-attention","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"gated-delta-networks-improving-mamba2-with","repo_url":"https://github.com/sustcsonglin/flash-linear-attention","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"gated-delta-networks-improving-mamba2-with","repo_url":"https://github.com/torchrwkv/flash-linear-attention","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"common-sense-reasoning","task_name":"Common Sense Reasoning"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"long-context-understanding","task_name":"Long-Context Understanding"},{"task_slug":"retrieval","task_name":"Retrieval"}],"methods":[{"method_slug":"attention","method_name":"Attention"},{"method_slug":"softmax","method_name":"Softmax"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2412.06464","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2412.06464"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/fla-org/flash-linear-attention","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sustcsonglin/flash-linear-attention","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/NVlabs/GatedDeltaNet","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/torchrwkv/flash-linear-attention","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":4,"ran_honours":1,"unverified":8},"by_repo_kind":{"official":{"samples":7,"ran":5,"repositories":1},"listed":{"samples":6,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":7,"samples":[{"code_sha256_prefix":"5ec949977c334f87","entry":"apply_rotary_emb_torch","repo":"NVlabs/GatedDeltaNet","repo_kind":"official","path":"lit_gpt/rotary.py","file_url":"https://github.com/NVlabs/GatedDeltaNet/blob/HEAD/lit_gpt/rotary.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5ec949977c334f87"}},{"code_sha256_prefix":"a30c7463cb6330d7","entry":"build_rope_cache","repo":"NVlabs/GatedDeltaNet","repo_kind":"official","path":"lit_gpt/model.py","file_url":"https://github.com/NVlabs/GatedDeltaNet/blob/HEAD/lit_gpt/model.py","link_basis":"plan_row","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a30c7463cb6330d7"}},{"code_sha256_prefix":"fa298642043e1282","entry":"code","repo":"NVlabs/GatedDeltaNet","repo_kind":"official","path":"lit_gpt/packed_dataset.py","file_url":"https://github.com/NVlabs/GatedDeltaNet/blob/HEAD/lit_gpt/packed_dataset.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fa298642043e1282"}},{"code_sha256_prefix":"c6283b7a1a1785ba","entry":"maybe_align","repo":"NVlabs/GatedDeltaNet","repo_kind":"official","path":"lit_gpt/rmsnorm.py","file_url":"https://github.com/NVlabs/GatedDeltaNet/blob/HEAD/lit_gpt/rmsnorm.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c6283b7a1a1785ba"}},{"code_sha256_prefix":"163292f43a3c861b","entry":"rotate_half","repo":"NVlabs/GatedDeltaNet","repo_kind":"official","path":"lit_gpt/rotary.py","file_url":"https://github.com/NVlabs/GatedDeltaNet/blob/HEAD/lit_gpt/rotary.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"163292f43a3c861b"}},{"code_sha256_prefix":"881e6b15cd91e16c","entry":"ceil_div","repo":"torchrwkv/flash-linear-attention","repo_kind":"listed","path":"fla/ops/log_linear_attn/chunk.py","file_url":"https://github.com/torchrwkv/flash-linear-attention/blob/HEAD/fla/ops/log_linear_attn/chunk.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"881e6b15cd91e16c"}},{"code_sha256_prefix":"35a90dba5e6339cb","entry":"construct_H_matrix","repo":"torchrwkv/flash-linear-attention","repo_kind":"listed","path":"fla/ops/log_linear_attn/naive.py","file_url":"https://github.com/torchrwkv/flash-linear-attention/blob/HEAD/fla/ops/log_linear_attn/naive.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"35a90dba5e6339cb"}},{"code_sha256_prefix":"f425d46b52d30143","entry":"construct_binary_level_mask","repo":"torchrwkv/flash-linear-attention","repo_kind":"listed","path":"fla/ops/log_linear_attn/chunk.py","file_url":"https://github.com/torchrwkv/flash-linear-attention/blob/HEAD/fla/ops/log_linear_attn/chunk.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f425d46b52d30143"}},{"code_sha256_prefix":"2eeea785ff36049a","entry":"construct_level_mask","repo":"torchrwkv/flash-linear-attention","repo_kind":"listed","path":"fla/ops/log_linear_attn/naive.py","file_url":"https://github.com/torchrwkv/flash-linear-attention/blob/HEAD/fla/ops/log_linear_attn/naive.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2eeea785ff36049a"}},{"code_sha256_prefix":"cf8f6cb38ea274f7","entry":"create_block","repo":"NVlabs/GatedDeltaNet","repo_kind":"official","path":"lit_gpt/model.py","file_url":"https://github.com/NVlabs/GatedDeltaNet/blob/HEAD/lit_gpt/model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"cf8f6cb38ea274f7"}},{"code_sha256_prefix":"3f3d25ea807f6d5c","entry":"get_flops_available","repo":"NVlabs/GatedDeltaNet","repo_kind":"official","path":"lit_gpt/speed_monitor.py","file_url":"https://github.com/NVlabs/GatedDeltaNet/blob/HEAD/lit_gpt/speed_monitor.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3f3d25ea807f6d5c"}},{"code_sha256_prefix":"2196376162089b22","entry":"level_lut","repo":"torchrwkv/flash-linear-attention","repo_kind":"listed","path":"fla/ops/log_linear_attn/chunk.py","file_url":"https://github.com/torchrwkv/flash-linear-attention/blob/HEAD/fla/ops/log_linear_attn/chunk.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2196376162089b22"}},{"code_sha256_prefix":"f0e873ec510a64bc","entry":"segsum","repo":"torchrwkv/flash-linear-attention","repo_kind":"listed","path":"fla/ops/log_linear_attn/naive.py","file_url":"https://github.com/torchrwkv/flash-linear-attention/blob/HEAD/fla/ops/log_linear_attn/naive.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f0e873ec510a64bc"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}