{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2605-08696","title":"Structured Recurrent Mixers for Massively Parallelized Sequence Generation","arxiv_id":"2605.08696","date":"2026-05-09","proceeding":null,"authors":["Benjamin L. Badger"],"abstract":"Over the last two decades, language modeling has experienced a shift from the use of predominantly recurrent architectures that process tokens sequentially during training and inference to non-recurrent models that process sequence elements in parallel during training, which results in greater training efficiency and stability at the expense of lower inference throughput. Here we introduce the Structured Recurrent Mixer, an architecture that allows for algebraic conversion between a sequence parallel representation at train time and a recurrent representation at inference, notably without the need for specialized kernels or device-specific memory management. We show experimentally that this dual representation allows for greater training efficiency, higher input information capacity, and larger inference throughput and concurrency when compared to other linear complexity models. We postulate that recurrent models are poorly suited to extended sequence length scaling for information-rich inputs typical of language, but are well suited to scaling in the sample (batch) dimension due to their constant memory per sample. We provide Mojo/MAX inference implementations of SRMs exhibiting 12x the throughput and 170x the concurrency of similarly powerful Transformers inferenced on vLLM, increases characteristic of Pytorch implementations resulting in a 30\\% increase in compute-constant GSM8k Pass@k. We conclude by demonstrating that SRMs are effective reinforcement learning training candidates.","url_abs":"https://arxiv.org/abs/2605.08696","url_pdf":"https://arxiv.org/pdf/2605.08696","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2605.08696","atlas_url":"https://app.syntology.ai/?focus=2605.08696","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2605.08696"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/blbadger/lm-evaluation-harness","reach":null}],"summary":{"ran":8,"unverified":3},"by_repo_kind":{"found_in_text":{"samples":11,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"bacb928cb85a8d51","entry":"ColRepeatCausalLinear","repo":"blbadger/lm-evaluation-harness","repo_kind":"found_in_text","path":"lm_eval/models/srm_model.py","file_url":"https://github.com/blbadger/lm-evaluation-harness/blob/HEAD/lm_eval/models/srm_model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bacb928cb85a8d51"}},{"code_sha256_prefix":"1c796bd7904ae352","entry":"CombinedRepeatCausalLinear","repo":"blbadger/lm-evaluation-harness","repo_kind":"found_in_text","path":"lm_eval/models/srm_model.py","file_url":"https://github.com/blbadger/lm-evaluation-harness/blob/HEAD/lm_eval/models/srm_model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1c796bd7904ae352"}},{"code_sha256_prefix":"a08b8b9498759559","entry":"HeadedRepeatCausalLinear","repo":"blbadger/lm-evaluation-harness","repo_kind":"found_in_text","path":"lm_eval/models/srm_model.py","file_url":"https://github.com/blbadger/lm-evaluation-harness/blob/HEAD/lm_eval/models/srm_model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a08b8b9498759559"}},{"code_sha256_prefix":"1e043f786bcb9c3c","entry":"KernelRepeatLinear","repo":"blbadger/lm-evaluation-harness","repo_kind":"found_in_text","path":"lm_eval/models/srm_model.py","file_url":"https://github.com/blbadger/lm-evaluation-harness/blob/HEAD/lm_eval/models/srm_model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1e043f786bcb9c3c"}},{"code_sha256_prefix":"af46e67d9ada1a52","entry":"MixedRepeatHeads","repo":"blbadger/lm-evaluation-harness","repo_kind":"found_in_text","path":"lm_eval/models/srm_model.py","file_url":"https://github.com/blbadger/lm-evaluation-harness/blob/HEAD/lm_eval/models/srm_model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"af46e67d9ada1a52"}},{"code_sha256_prefix":"b677012c85ad1913","entry":"ParallelRepeatHeads","repo":"blbadger/lm-evaluation-harness","repo_kind":"found_in_text","path":"lm_eval/models/srm_model.py","file_url":"https://github.com/blbadger/lm-evaluation-harness/blob/HEAD/lm_eval/models/srm_model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b677012c85ad1913"}},{"code_sha256_prefix":"90b0ed5842d1a6a8","entry":"RepeatCausalLinear","repo":"blbadger/lm-evaluation-harness","repo_kind":"found_in_text","path":"lm_eval/models/srm_model.py","file_url":"https://github.com/blbadger/lm-evaluation-harness/blob/HEAD/lm_eval/models/srm_model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"90b0ed5842d1a6a8"}},{"code_sha256_prefix":"67e0fe4c73bec386","entry":"RowRepeatCausalLinear","repo":"blbadger/lm-evaluation-harness","repo_kind":"found_in_text","path":"lm_eval/models/srm_model.py","file_url":"https://github.com/blbadger/lm-evaluation-harness/blob/HEAD/lm_eval/models/srm_model.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"67e0fe4c73bec386"}},{"code_sha256_prefix":"af3cfa3cd6c4d580","entry":"MLPMixer","repo":"blbadger/lm-evaluation-harness","repo_kind":"found_in_text","path":"lm_eval/models/srm_model.py","file_url":"https://github.com/blbadger/lm-evaluation-harness/blob/HEAD/lm_eval/models/srm_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"af3cfa3cd6c4d580"}},{"code_sha256_prefix":"82f552f2e51345d1","entry":"MixerBlock","repo":"blbadger/lm-evaluation-harness","repo_kind":"found_in_text","path":"lm_eval/models/srm_model.py","file_url":"https://github.com/blbadger/lm-evaluation-harness/blob/HEAD/lm_eval/models/srm_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"82f552f2e51345d1"}},{"code_sha256_prefix":"5b043b13bb1d8051","entry":"RepeatHeads","repo":"blbadger/lm-evaluation-harness","repo_kind":"found_in_text","path":"lm_eval/models/srm_model.py","file_url":"https://github.com/blbadger/lm-evaluation-harness/blob/HEAD/lm_eval/models/srm_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5b043b13bb1d8051"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.LG","source":"arxiv_2026.jsonl"},"syntology_extracted_results":null}