{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/rublimp-russian-benchmark-of-linguistic","title":"RuBLiMP: Russian Benchmark of Linguistic Minimal Pairs","arxiv_id":"2406.19232","date":"2024-06-27","proceeding":null,"authors":["Ekaterina Taktasheva","Maxim Bazhukov","Kirill Koncha","Alena Fenogenova","Ekaterina Artemova","Vladislav Mikhailov"],"abstract":"Minimal pairs are a well-established approach to evaluating the grammatical knowledge of language models. However, existing resources for minimal pairs address a limited number of languages and lack diversity of language-specific grammatical phenomena. This paper introduces the Russian Benchmark of Linguistic Minimal Pairs (RuBLiMP), which includes 45k pairs of sentences that differ in grammaticality and isolate a morphological, syntactic, or semantic phenomenon. In contrast to existing benchmarks of linguistic minimal pairs, RuBLiMP is created by applying linguistic perturbations to automatically annotated sentences from open text corpora and carefully curating test data. We describe the data collection protocol and present the results of evaluating 25 language models in various scenarios. We find that the widely used language models for Russian are sensitive to morphological and agreement-oriented contrasts but fall behind humans on phenomena requiring understanding of structural relations, negation, transitivity, and tense. RuBLiMP, the codebase, and other materials are publicly available.","url_abs":"https://arxiv.org/abs/2406.19232v3","url_pdf":"https://arxiv.org/pdf/2406.19232v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"rublimp-russian-benchmark-of-linguistic","repo_url":"https://github.com/russiannlp/rublimp","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"diversity","task_name":"Diversity"},{"task_slug":"negation","task_name":"Negation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2406.19232","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.19232"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/RussianNLP/RuBLiMP","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/natasha/slovnet","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":14,"unverified":2},"by_repo_kind":{"official":{"samples":6,"ran":6,"repositories":1},"found_in_text":{"samples":10,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"8e7a3b391779e819","entry":"bert_chunks","repo":"natasha/slovnet","repo_kind":"found_in_text","path":"slovnet/bert.py","file_url":"https://github.com/natasha/slovnet/blob/HEAD/slovnet/bert.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8e7a3b391779e819"}},{"code_sha256_prefix":"b11c77e1e013d613","entry":"capitalize_indexes","repo":"RussianNLP/RuBLiMP","repo_kind":"official","path":"src/utils/utils.py","file_url":"https://github.com/RussianNLP/RuBLiMP/blob/HEAD/src/utils/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b11c77e1e013d613"}},{"code_sha256_prefix":"beffc6b278a14d3e","entry":"capitalize_word","repo":"RussianNLP/RuBLiMP","repo_kind":"official","path":"src/utils/utils.py","file_url":"https://github.com/RussianNLP/RuBLiMP/blob/HEAD/src/utils/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"beffc6b278a14d3e"}},{"code_sha256_prefix":"78ee6585f194e2b4","entry":"find_nouns_man_woman","repo":"RussianNLP/RuBLiMP","repo_kind":"official","path":"src/utils/data_loaders.py","file_url":"https://github.com/RussianNLP/RuBLiMP/blob/HEAD/src/utils/data_loaders.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"78ee6585f194e2b4"}},{"code_sha256_prefix":"ffbbf48436ea48e9","entry":"find_nouns_prof_commongender","repo":"RussianNLP/RuBLiMP","repo_kind":"official","path":"src/utils/data_loaders.py","file_url":"https://github.com/RussianNLP/RuBLiMP/blob/HEAD/src/utils/data_loaders.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ffbbf48436ea48e9"}},{"code_sha256_prefix":"d9def9c954ea4370","entry":"format_bio","repo":"natasha/slovnet","repo_kind":"found_in_text","path":"slovnet/bio.py","file_url":"https://github.com/natasha/slovnet/blob/HEAD/slovnet/bio.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d9def9c954ea4370"}},{"code_sha256_prefix":"60009883d53fc392","entry":"format_conll_tag","repo":"natasha/slovnet","repo_kind":"found_in_text","path":"slovnet/conll.py","file_url":"https://github.com/natasha/slovnet/blob/HEAD/slovnet/conll.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"60009883d53fc392"}},{"code_sha256_prefix":"1b17728d03301242","entry":"getcapital","repo":"RussianNLP/RuBLiMP","repo_kind":"official","path":"src/utils/utils.py","file_url":"https://github.com/RussianNLP/RuBLiMP/blob/HEAD/src/utils/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1b17728d03301242"}},{"code_sha256_prefix":"ea0488d1412c27b2","entry":"load_json","repo":"natasha/slovnet","repo_kind":"found_in_text","path":"slovnet/io.py","file_url":"https://github.com/natasha/slovnet/blob/HEAD/slovnet/io.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ea0488d1412c27b2"}},{"code_sha256_prefix":"dbe2a14f4e641a48","entry":"load_vocab","repo":"RussianNLP/RuBLiMP","repo_kind":"official","path":"src/utils/data_loaders.py","file_url":"https://github.com/RussianNLP/RuBLiMP/blob/HEAD/src/utils/data_loaders.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"dbe2a14f4e641a48"}},{"code_sha256_prefix":"e686b8ce6606b5a9","entry":"parse_bio","repo":"natasha/slovnet","repo_kind":"found_in_text","path":"slovnet/bio.py","file_url":"https://github.com/natasha/slovnet/blob/HEAD/slovnet/bio.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e686b8ce6606b5a9"}},{"code_sha256_prefix":"b0d5d5c95bce546f","entry":"parse_conll_tag","repo":"natasha/slovnet","repo_kind":"found_in_text","path":"slovnet/conll.py","file_url":"https://github.com/natasha/slovnet/blob/HEAD/slovnet/conll.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b0d5d5c95bce546f"}},{"code_sha256_prefix":"ae02bed8c6ac0e37","entry":"safe_wordpiece","repo":"natasha/slovnet","repo_kind":"found_in_text","path":"slovnet/bert.py","file_url":"https://github.com/natasha/slovnet/blob/HEAD/slovnet/bert.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ae02bed8c6ac0e37"}},{"code_sha256_prefix":"70e55146aa0bea49","entry":"wordpiece","repo":"natasha/slovnet","repo_kind":"found_in_text","path":"slovnet/bert.py","file_url":"https://github.com/natasha/slovnet/blob/HEAD/slovnet/bert.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"70e55146aa0bea49"}},{"code_sha256_prefix":"2083fcb410f76cfd","entry":"flatten_cross_entropy","repo":"natasha/slovnet","repo_kind":"found_in_text","path":"slovnet/loss.py","file_url":"https://github.com/natasha/slovnet/blob/HEAD/slovnet/loss.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2083fcb410f76cfd"}},{"code_sha256_prefix":"6b1d272e499dcae8","entry":"masked_flatten_cross_entropy","repo":"natasha/slovnet","repo_kind":"found_in_text","path":"slovnet/loss.py","file_url":"https://github.com/natasha/slovnet/blob/HEAD/slovnet/loss.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6b1d272e499dcae8"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}