{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/pl-mteb-polish-massive-text-embedding","title":"PL-MTEB: Polish Massive Text Embedding Benchmark","arxiv_id":"2405.10138","date":"2024-05-16","proceeding":null,"authors":["Rafał Poświata","Sławomir Dadas","Michał Perełkiewicz"],"abstract":"In this paper, we introduce the Polish Massive Text Embedding Benchmark (PL-MTEB), a comprehensive benchmark for text embeddings in Polish. The PL-MTEB consists of 28 diverse NLP tasks from 5 task types. We adapted the tasks based on previously used datasets by the Polish NLP community. In addition, we created a new PLSC (Polish Library of Science Corpus) dataset consisting of titles and abstracts of scientific publications in Polish, which was used as the basis for two novel clustering tasks. We evaluated 15 publicly available models for text embedding, including Polish and multilingual ones, and collected detailed results for individual tasks and aggregated results for each task type and the entire benchmark. PL-MTEB comes with open-source code at https://github.com/rafalposwiata/pl-mteb.","url_abs":"https://arxiv.org/abs/2405.10138v1","url_pdf":"https://arxiv.org/pdf/2405.10138v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"pl-mteb-polish-massive-text-embedding","repo_url":"https://github.com/rafalposwiata/pl-mteb","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}}],"tasks":[],"methods":[{"method_slug":null,"method_name":"Library"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2405.10138","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.10138"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/rafalposwiata/pl-mteb","reach":{"status":"ok"}}],"summary":{"ran":7},"by_repo_kind":{"official":{"samples":7,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":7,"samples":[{"code_sha256_prefix":"6d0656f5c82bdaed","entry":"from_dict","repo":"rafalposwiata/pl-mteb","repo_kind":"official","path":"utils.py","file_url":"https://github.com/rafalposwiata/pl-mteb/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6d0656f5c82bdaed"}},{"code_sha256_prefix":"c9331b22fb8aa88a","entry":"get_first_not_none","repo":"rafalposwiata/pl-mteb","repo_kind":"official","path":"utils.py","file_url":"https://github.com/rafalposwiata/pl-mteb/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c9331b22fb8aa88a"}},{"code_sha256_prefix":"d3d2208d6468ed08","entry":"normalize_text","repo":"rafalposwiata/pl-mteb","repo_kind":"official","path":"tasks/preparation/cleaning.py","file_url":"https://github.com/rafalposwiata/pl-mteb/blob/HEAD/tasks/preparation/cleaning.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d3d2208d6468ed08"}},{"code_sha256_prefix":"1f5e1260223c2c07","entry":"remove_all_spaces","repo":"rafalposwiata/pl-mteb","repo_kind":"official","path":"tasks/preparation/cleaning.py","file_url":"https://github.com/rafalposwiata/pl-mteb/blob/HEAD/tasks/preparation/cleaning.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1f5e1260223c2c07"}},{"code_sha256_prefix":"6d301fb7295cd462","entry":"remove_multiple_spaces","repo":"rafalposwiata/pl-mteb","repo_kind":"official","path":"tasks/preparation/cleaning.py","file_url":"https://github.com/rafalposwiata/pl-mteb/blob/HEAD/tasks/preparation/cleaning.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6d301fb7295cd462"}},{"code_sha256_prefix":"6ab39d36e9687451","entry":"update_datasets_versions","repo":"rafalposwiata/pl-mteb","repo_kind":"official","path":"models/prompts.py","file_url":"https://github.com/rafalposwiata/pl-mteb/blob/HEAD/models/prompts.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6ab39d36e9687451"}},{"code_sha256_prefix":"6c6ed070838221c2","entry":"wrap_with_marker","repo":"rafalposwiata/pl-mteb","repo_kind":"official","path":"results_summarization.py","file_url":"https://github.com/rafalposwiata/pl-mteb/blob/HEAD/results_summarization.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6c6ed070838221c2"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}