{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/revisiting-recurrent-reinforcement-learning","title":"Recurrent Reinforcement Learning with Memoroids","arxiv_id":"2402.09900","date":"2024-02-15","proceeding":null,"authors":["Steven Morad","Chris Lu","Ryan Kortvelesy","Stephan Liwicki","Jakob Foerster","Amanda Prorok"],"abstract":"Memory models such as Recurrent Neural Networks (RNNs) and Transformers address Partially Observable Markov Decision Processes (POMDPs) by mapping trajectories to latent Markov states. Neither model scales particularly well to long sequences, especially compared to an emerging class of memory models called Linear Recurrent Models. We discover that the recurrent update of these models resembles a monoid, leading us to reformulate existing models using a novel monoid-based framework that we call memoroids. We revisit the traditional approach to batching in recurrent reinforcement learning, highlighting theoretical and empirical deficiencies. We leverage memoroids to propose a batching method that improves sample efficiency, increases the return, and simplifies the implementation of recurrent loss functions in reinforcement learning.","url_abs":"https://arxiv.org/abs/2402.09900v3","url_pdf":"https://arxiv.org/pdf/2402.09900v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"revisiting-recurrent-reinforcement-learning","repo_url":"https://github.com/proroklab/memory-monoids","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"jax","reach":null}],"tasks":[{"task_slug":"reinforcement-learning","task_name":"Reinforcement Learning"},{"task_slug":"reinforcement-learning-2","task_name":"reinforcement-learning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2402.09900","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.09900"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/proroklab/memory-monoids","reach":null}],"summary":{"ran_honours":1,"ran_fixture":1,"ran_draft_wrong":4,"unverified":1},"by_repo_kind":{"official":{"samples":6,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":7,"samples":[{"code_sha256_prefix":"0212f0ba3cfe6d69","entry":"binary_operator_diag","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"0212f0ba3cfe6d69"}},{"code_sha256_prefix":"f1d67707c6650049","entry":"gamma_log_init","repo":"proroklab/memory-monoids","repo_kind":"official","path":"memory/lru.py","file_url":"https://github.com/proroklab/memory-monoids/blob/HEAD/memory/lru.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f1d67707c6650049"}},{"code_sha256_prefix":"702f95276a6ff829","entry":"matrix_init","repo":"proroklab/memory-monoids","repo_kind":"official","path":"memory/lru.py","file_url":"https://github.com/proroklab/memory-monoids/blob/HEAD/memory/lru.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"702f95276a6ff829"}},{"code_sha256_prefix":"f832dda1d4b9b1d8","entry":"nu_init","repo":"proroklab/memory-monoids","repo_kind":"official","path":"memory/lru.py","file_url":"https://github.com/proroklab/memory-monoids/blob/HEAD/memory/lru.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f832dda1d4b9b1d8"}},{"code_sha256_prefix":"5eb92a40d3fbcc07","entry":"theta_init","repo":"proroklab/memory-monoids","repo_kind":"official","path":"memory/lru.py","file_url":"https://github.com/proroklab/memory-monoids/blob/HEAD/memory/lru.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5eb92a40d3fbcc07"}},{"code_sha256_prefix":"be06105188b4bb18","entry":"wrapped_associative_update","repo":"proroklab/memory-monoids","repo_kind":"official","path":"memory/lru.py","file_url":"https://github.com/proroklab/memory-monoids/blob/HEAD/memory/lru.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"be06105188b4bb18"}},{"code_sha256_prefix":"871a68de4ff3cdef","entry":"LRU","repo":"proroklab/memory-monoids","repo_kind":"official","path":"memory/lru.py","file_url":"https://github.com/proroklab/memory-monoids/blob/HEAD/memory/lru.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"871a68de4ff3cdef"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}