{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/russiansuperglue-a-russian-language","title":"RussianSuperGLUE: A Russian Language Understanding Evaluation Benchmark","arxiv_id":"2010.15925","date":"2020-10-29","proceeding":"EMNLP 2020 11","authors":["Tatiana Shavrina","Alena Fenogenova","Anton Emelyanov","Denis Shevelev","Ekaterina Artemova","Valentin Malykh","Vladislav Mikhailov","Maria Tikhonova","Andrey Chertok","Andrey Evlampiev"],"abstract":"In this paper, we introduce an advanced Russian general language understanding evaluation benchmark -- RussianGLUE. Recent advances in the field of universal language models and transformers require the development of a methodology for their broad diagnostics and testing for general intellectual skills - detection of natural language inference, commonsense reasoning, ability to perform simple logical operations regardless of text subject or lexicon. For the first time, a benchmark of nine tasks, collected and organized analogically to the SuperGLUE methodology, was developed from scratch for the Russian language. We provide baselines, human level evaluation, an open-source framework for evaluating models (https://github.com/RussianNLP/RussianSuperGLUE), and an overall leaderboard of transformer models for the Russian language. Besides, we present the first results of comparing multilingual models in the adapted diagnostic test set and offer the first steps to further expanding or assessing state-of-the-art models independently of language.","url_abs":"https://arxiv.org/abs/2010.15925v2","url_pdf":"https://arxiv.org/pdf/2010.15925v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"russiansuperglue-a-russian-language","repo_url":"https://github.com/RussianNLP/RussianSuperGLUE","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":null},{"paper_slug":"russiansuperglue-a-russian-language","repo_url":"https://github.com/RussianNLP/MOROCCO","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"common-sense-reasoning","task_name":"Common Sense Reasoning"},{"task_slug":"diagnostic","task_name":"Diagnostic"},{"task_slug":"lexical-entailment","task_name":"Lexical Entailment"},{"task_slug":"logical-reasoning-question-ansering","task_name":"Logical Reasoning Question Answering"},{"task_slug":"natural-language-inference","task_name":"Natural Language Inference"},{"task_slug":"natural-language-understanding","task_name":"Natural Language Understanding"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"reading-comprehension","task_name":"Reading Comprehension"},{"task_slug":"word-sense-disambiguation","task_name":"Word Sense Disambiguation"}],"methods":[],"datasets_introduced":[{"slug":"danetqa","name":"DaNetQA","full_name":"Yes/no Question Answering Dataset for the Russian"},{"slug":"lidirus","name":"LiDiRus","full_name":"Linguistic Diagnostic for Russian"},{"slug":"muserc","name":"MuSeRC","full_name":"Russian Multi-Sentence Reading Comprehension"},{"slug":"parus","name":"PARus","full_name":"Choice of Plausible Alternatives for Russian language"},{"slug":"rcb","name":"RCB","full_name":"Russian Commitment Bank"},{"slug":"rwsd","name":"RWSD","full_name":"The Winograd Schema Challenge (Russian)"},{"slug":"rucos","name":"RuCoS","full_name":"Russian Reading Comprehension with Commonsense Reasoning"},{"slug":"terra","name":"TERRa","full_name":"Textual Entailment Recognition for Russian"}],"methods_introduced":[],"results":[{"leaderboard":"/sota/common-sense-reasoning-on-parus","task":"Common Sense Reasoning","dataset":"PARus","model":"Human Benchmark","rank_in_archive_order":1,"of":22,"metrics":{"Accuracy":"0.982"},"uses_additional_data":false},{"leaderboard":"/sota/common-sense-reasoning-on-parus","task":"Common Sense Reasoning","dataset":"PARus","model":"Baseline TF-IDF1.1","rank_in_archive_order":19,"of":22,"metrics":{"Accuracy":"0.486"},"uses_additional_data":false},{"leaderboard":"/sota/common-sense-reasoning-on-rwsd","task":"Common Sense Reasoning","dataset":"RWSD","model":"Baseline TF-IDF1.1","rank_in_archive_order":7,"of":22,"metrics":{"Accuracy":"0.662"},"uses_additional_data":false},{"leaderboard":"/sota/common-sense-reasoning-on-rwsd","task":"Common Sense Reasoning","dataset":"RWSD","model":"Human Benchmark","rank_in_archive_order":22,"of":22,"metrics":{"Accuracy":"0.84"},"uses_additional_data":false},{"leaderboard":"/sota/common-sense-reasoning-on-rucos","task":"Common Sense Reasoning","dataset":"RuCoS","model":"Human Benchmark","rank_in_archive_order":1,"of":22,"metrics":{"Average F1":"0.93","EM ":"0.89"},"uses_additional_data":false},{"leaderboard":"/sota/common-sense-reasoning-on-rucos","task":"Common Sense Reasoning","dataset":"RuCoS","model":"Baseline TF-IDF1.1","rank_in_archive_order":16,"of":22,"metrics":{"Average F1":"0.26","EM ":"0.252"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-lidirus","task":"Natural Language Inference","dataset":"LiDiRus","model":"Human Benchmark","rank_in_archive_order":1,"of":22,"metrics":{"MCC":"0.626"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-lidirus","task":"Natural Language Inference","dataset":"LiDiRus","model":"Baseline TF-IDF1.1","rank_in_archive_order":17,"of":22,"metrics":{"MCC":"0.06"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-rcb","task":"Natural Language Inference","dataset":"RCB","model":"Human Benchmark","rank_in_archive_order":1,"of":22,"metrics":{"Accuracy":"0.702","Average F1":"0.68"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-rcb","task":"Natural Language Inference","dataset":"RCB","model":"Baseline TF-IDF1.1","rank_in_archive_order":21,"of":22,"metrics":{"Accuracy":"0.441","Average F1":"0.301"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-terra","task":"Natural Language Inference","dataset":"TERRa","model":"Human Benchmark","rank_in_archive_order":1,"of":22,"metrics":{"Accuracy":"0.92"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-terra","task":"Natural Language Inference","dataset":"TERRa","model":"Baseline TF-IDF1.1","rank_in_archive_order":22,"of":22,"metrics":{"Accuracy":"0.471"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-danetqa","task":"Question Answering","dataset":"DaNetQA","model":"Human Benchmark","rank_in_archive_order":2,"of":22,"metrics":{"Accuracy":"0.915"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-danetqa","task":"Question Answering","dataset":"DaNetQA","model":"Baseline TF-IDF1.1","rank_in_archive_order":16,"of":22,"metrics":{"Accuracy":"0.621"},"uses_additional_data":false},{"leaderboard":"/sota/reading-comprehension-on-muserc","task":"Reading Comprehension","dataset":"MuSeRC","model":"Human Benchmark","rank_in_archive_order":5,"of":22,"metrics":{"Average F1":"0.806","EM ":"0.42"},"uses_additional_data":false},{"leaderboard":"/sota/reading-comprehension-on-muserc","task":"Reading Comprehension","dataset":"MuSeRC","model":"Baseline TF-IDF1.1","rank_in_archive_order":20,"of":22,"metrics":{"Average F1":"0.587","EM ":" 0.242"},"uses_additional_data":false},{"leaderboard":"/sota/word-sense-disambiguation-on-russe","task":"Word Sense Disambiguation","dataset":"RUSSE","model":"Human Benchmark","rank_in_archive_order":1,"of":22,"metrics":{"Accuracy":"0.805"},"uses_additional_data":false},{"leaderboard":"/sota/word-sense-disambiguation-on-russe","task":"Word Sense Disambiguation","dataset":"RUSSE","model":"Baseline TF-IDF1.1","rank_in_archive_order":20,"of":22,"metrics":{"Accuracy":"0.57"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2010.15925","atlas_url":"https://app.syntology.ai/?focus=2010.15925","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.15925"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/RussianNLP/MOROCCO","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/RussianNLP/RussianSuperGLUE","reach":null}],"summary":{"ran_honours":1},"by_repo_kind":{"official":{"samples":1,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"85e4a5947d2004ea","entry":"parse_write_preds_arg","repo":"RussianNLP/RussianSuperGLUE","repo_kind":"official","path":"jiant-russian-v2/evaluate.py","file_url":"https://github.com/RussianNLP/RussianSuperGLUE/blob/HEAD/jiant-russian-v2/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"85e4a5947d2004ea"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}