{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/rwkv-7-goose-with-expressive-dynamic-state","title":"RWKV-7 \"Goose\" with Expressive Dynamic State Evolution","arxiv_id":"2503.14456","date":"2025-03-18","proceeding":null,"authors":["Bo Peng","Ruichong Zhang","Daniel Goldstein","Eric Alcaide","Xingjian Du","Haowen Hou","Jiaju Lin","Jiaxing Liu","Janna Lu","William Merrill","Guangyu Song","Kaifeng Tan","Saiteja Utpala","Nathan Wilce","Johan S. Wind","Tianyi Wu","Daniel Wuttke","Christian Zhou-Zheng"],"abstract":"We present RWKV-7 \"Goose\", a new sequence modeling architecture with constant memory usage and constant inference time per token. Despite being trained on dramatically fewer tokens than other top models, our 2.9 billion parameter language model achieves a new 3B SoTA on multilingual tasks and matches the current 3B SoTA on English language downstream performance. RWKV-7 introduces a newly generalized formulation of the delta rule with vector-valued gating and in-context learning rates, as well as a relaxed value replacement rule. We show that RWKV-7 can perform state tracking and recognize all regular languages, while retaining parallelizability of training. This exceeds the capabilities of Transformers under standard complexity conjectures, which are limited to $\\mathsf{TC}^0$. To demonstrate RWKV-7's language modeling capability, we also present an extended open source 3.1 trillion token multilingual corpus, and train four RWKV-7 models ranging from 0.19 billion to 2.9 billion parameters on this dataset. To foster openness, reproduction, and adoption, we release our models and dataset component listing at https://huggingface.co/RWKV, and our training and inference code at https://github.com/RWKV/RWKV-LM all under the Apache 2.0 License.","url_abs":"https://arxiv.org/abs/2503.14456v2","url_pdf":"https://arxiv.org/pdf/2503.14456v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"rwkv-7-goose-with-expressive-dynamic-state","repo_url":"https://github.com/fla-org/flash-linear-attention","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"rwkv-7-goose-with-expressive-dynamic-state","repo_url":"https://github.com/rwkv/rwkv-lm","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":null},{"paper_slug":"rwkv-7-goose-with-expressive-dynamic-state","repo_url":"https://github.com/blinkdl/chatrwkv","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"in-context-learning","task_name":"In-Context Learning"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2503.14456","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2503.14456"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/fla-org/flash-linear-attention","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/blinkdl/chatrwkv","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/rwkv/rwkv-lm","reach":null}],"summary":{"ran_fixture":1,"ran_draft_wrong":1,"unverified":1},"by_repo_kind":{"official":{"samples":1,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":2,"samples":[{"code_sha256_prefix":"852ba565f9d4f477","entry":"RWKV_x070_TMix_one","repo":"rwkv/rwkv-lm","repo_kind":"official","path":"RWKV-v7/rwkv_v7_demo_fast.py","file_url":"https://github.com/rwkv/rwkv-lm/blob/HEAD/RWKV-v7/rwkv_v7_demo_fast.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"852ba565f9d4f477"}},{"code_sha256_prefix":"07a576b3352ec5ac","entry":"time_mixing__","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"07a576b3352ec5ac"}},{"code_sha256_prefix":"a43aabdbc73f2f29","entry":"channel_mixing__","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"a43aabdbc73f2f29"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}