{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2607-21447","title":"RUMBA: Russian User Memory Benchmark","arxiv_id":"2607.21447","date":"2026-07-23","proceeding":null,"authors":["Elizaveta Shevtsova","Inna Glebkina","Mark Baushenko","Pavel Gulyaev","Alena Fenogenova"],"abstract":"The ability to handle long-term memory in LLMs is becoming increasingly critical, yet existing benchmarks remain English-centric and rely on aggregate retrieval metrics, failing to capture interactions between long-range context, temporal information, and reasoning. To address this, we introduce RUMBA (Russian User Memory BenchmArk) - a new benchmark for long-term conversational memory that provides a fine-grained taxonomy of memory-centric question types and a unified methodology accounting for semantic type, session scope, temporal reasoning, and the explicitness of temporal expressions. RUMBA consists of timestamped user-assistant dialogues with QA pairs requiring retrieval, combination, and reasoning across sessions. While designed for Russian, we also provide an aligned English subset under the same methodology. We evaluate contemporary memory systems and long-context models, and show how RUMBA serves as a diagnostic tool to analyze model behavior across benchmark slices and identify strengths and failure modes of different memory mechanisms.","url_abs":"https://arxiv.org/abs/2607.21447","url_pdf":"https://arxiv.org/pdf/2607.21447","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2607.21447","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2607.21447"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/ai-forever/RUMBA","reach":null}],"summary":{"ran":13,"unverified":1},"by_repo_kind":{"found_in_text":{"samples":14,"ran":13,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"4926ea8b47ad1a6c","entry":"build_cat_stats","repo":"ai-forever/RUMBA","repo_kind":"found_in_text","path":"evaluation/metric_analytics.py","file_url":"https://github.com/ai-forever/RUMBA/blob/HEAD/evaluation/metric_analytics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4926ea8b47ad1a6c"}},{"code_sha256_prefix":"06fe12dbaea2bee7","entry":"conversation_to_pairs","repo":"ai-forever/RUMBA","repo_kind":"found_in_text","path":"evaluation/run_experiments_add.py","file_url":"https://github.com/ai-forever/RUMBA/blob/HEAD/evaluation/run_experiments_add.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"06fe12dbaea2bee7"}},{"code_sha256_prefix":"2cc374dddd30bcf3","entry":"conversation_to_payload","repo":"ai-forever/RUMBA","repo_kind":"found_in_text","path":"evaluation/run_experiments_add.py","file_url":"https://github.com/ai-forever/RUMBA/blob/HEAD/evaluation/run_experiments_add.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2cc374dddd30bcf3"}},{"code_sha256_prefix":"ea3bf2870cd5128d","entry":"extract_judge_feedback","repo":"ai-forever/RUMBA","repo_kind":"found_in_text","path":"evaluation/lighteval_ext/custom_tasks/rumba_en.py","file_url":"https://github.com/ai-forever/RUMBA/blob/HEAD/evaluation/lighteval_ext/custom_tasks/rumba_en.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ea3bf2870cd5128d"}},{"code_sha256_prefix":"fd081534bb7e149e","entry":"extract_timestamp","repo":"ai-forever/RUMBA","repo_kind":"found_in_text","path":"evaluation/users_stats_query_qdrant.py","file_url":"https://github.com/ai-forever/RUMBA/blob/HEAD/evaluation/users_stats_query_qdrant.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fd081534bb7e149e"}},{"code_sha256_prefix":"689115f86c329ff8","entry":"extract_user_index","repo":"ai-forever/RUMBA","repo_kind":"found_in_text","path":"evaluation/users_stats_query_chroma.py","file_url":"https://github.com/ai-forever/RUMBA/blob/HEAD/evaluation/users_stats_query_chroma.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"689115f86c329ff8"}},{"code_sha256_prefix":"b8589dcf97d6f447","entry":"get_count","repo":"ai-forever/RUMBA","repo_kind":"found_in_text","path":"evaluation/users_stats_query_chroma.py","file_url":"https://github.com/ai-forever/RUMBA/blob/HEAD/evaluation/users_stats_query_chroma.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b8589dcf97d6f447"}},{"code_sha256_prefix":"d4e229fbe4dfa7ee","entry":"get_latest_dir","repo":"ai-forever/RUMBA","repo_kind":"found_in_text","path":"evaluation/metric_analytics.py","file_url":"https://github.com/ai-forever/RUMBA/blob/HEAD/evaluation/metric_analytics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d4e229fbe4dfa7ee"}},{"code_sha256_prefix":"1bc343e067430b22","entry":"is_valid_iso","repo":"ai-forever/RUMBA","repo_kind":"found_in_text","path":"evaluation/users_stats_query_qdrant.py","file_url":"https://github.com/ai-forever/RUMBA/blob/HEAD/evaluation/users_stats_query_qdrant.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1bc343e067430b22"}},{"code_sha256_prefix":"e59cdd555ce8d27e","entry":"list_collection_names","repo":"ai-forever/RUMBA","repo_kind":"found_in_text","path":"evaluation/users_stats_query_chroma.py","file_url":"https://github.com/ai-forever/RUMBA/blob/HEAD/evaluation/users_stats_query_chroma.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e59cdd555ce8d27e"}},{"code_sha256_prefix":"c38f105fc182acf1","entry":"load_data","repo":"ai-forever/RUMBA","repo_kind":"found_in_text","path":"evaluation/run_experiments_add.py","file_url":"https://github.com/ai-forever/RUMBA/blob/HEAD/evaluation/run_experiments_add.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c38f105fc182acf1"}},{"code_sha256_prefix":"a6d9ba4fc8a34593","entry":"load_details_df","repo":"ai-forever/RUMBA","repo_kind":"found_in_text","path":"evaluation/metric_analytics.py","file_url":"https://github.com/ai-forever/RUMBA/blob/HEAD/evaluation/metric_analytics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a6d9ba4fc8a34593"}},{"code_sha256_prefix":"99ca46ea75be9332","entry":"process_judge_response","repo":"ai-forever/RUMBA","repo_kind":"found_in_text","path":"evaluation/lighteval_ext/custom_tasks/rumba_ru.py","file_url":"https://github.com/ai-forever/RUMBA/blob/HEAD/evaluation/lighteval_ext/custom_tasks/rumba_ru.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"99ca46ea75be9332"}},{"code_sha256_prefix":"cef73a9b4b26450b","entry":"print_section","repo":"ai-forever/RUMBA","repo_kind":"found_in_text","path":"evaluation/users_stats_query_neo4j.py","file_url":"https://github.com/ai-forever/RUMBA/blob/HEAD/evaluation/users_stats_query_neo4j.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cef73a9b4b26450b"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.CL","source":"arxiv_2026.jsonl"},"syntology_extracted_results":null}