{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/wikimatrix-mining-135m-parallel-sentences-in","title":"WikiMatrix: Mining 135M Parallel Sentences in 1620 Language Pairs from Wikipedia","arxiv_id":"1907.05791","date":"2019-07-10","proceeding":"EACL 2021 2","authors":["Holger Schwenk","Vishrav Chaudhary","Shuo Sun","Hongyu Gong","Francisco Guzmán"],"abstract":"We present an approach based on multilingual sentence embeddings to automatically extract parallel sentences from the content of Wikipedia articles in 85 languages, including several dialects or low-resource languages. We do not limit the the extraction process to alignments with English, but systematically consider all possible language pairs. In total, we are able to extract 135M parallel sentences for 1620 different language pairs, out of which only 34M are aligned with English. This corpus of parallel sentences is freely available at https://github.com/facebookresearch/LASER/tree/master/tasks/WikiMatrix. To get an indication on the quality of the extracted bitexts, we train neural MT baseline systems on the mined data only for 1886 languages pairs, and evaluate them on the TED corpus, achieving strong BLEU scores for many language pairs. The WikiMatrix bitexts seem to be particularly interesting to train MT systems between distant languages without the need to pivot through English.","url_abs":"https://arxiv.org/abs/1907.05791v2","url_pdf":"https://arxiv.org/pdf/1907.05791v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"wikimatrix-mining-135m-parallel-sentences-in","repo_url":"https://github.com/facebookresearch/LASER","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"wikimatrix-mining-135m-parallel-sentences-in","repo_url":"https://github.com/EugeneSel/EUMT","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"wikimatrix-mining-135m-parallel-sentences-in","repo_url":"https://github.com/RachelChen1116/WikiNLI","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"wikimatrix-mining-135m-parallel-sentences-in","repo_url":"https://github.com/kmkwon94/ainize-laser","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}},{"paper_slug":"wikimatrix-mining-135m-parallel-sentences-in","repo_url":"https://github.com/mmcux/de-nds-translation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"wikimatrix-mining-135m-parallel-sentences-in","repo_url":"https://github.com/prabhakar267/LASER-improved","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}}],"tasks":[{"task_slug":"articles","task_name":"Articles"},{"task_slug":"sentence","task_name":"Sentence"},{"task_slug":"sentence-embeddings","task_name":"Sentence Embeddings"}],"methods":[],"datasets_introduced":[{"slug":"wikimatrix","name":"WikiMatrix","full_name":""}],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=1907.05791","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1907.05791"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mmcux/de-nds-translation","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/LASER","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/EugeneSel/EUMT","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/RachelChen1116/WikiNLI","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kmkwon94/ainize-laser","reach":{"status":"ok","spdx":"NOASSERTION"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/prabhakar267/LASER-improved","reach":{"status":"ok","spdx":"NOASSERTION"}}],"summary":{"ran_honours":1,"unverified":2},"by_repo_kind":{"listed":{"samples":3,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"f6b944f50d3f15ae","entry":"count_parameters","repo":"mmcux/de-nds-translation","repo_kind":"listed","path":"translation_sentencepiece.py","file_url":"https://github.com/mmcux/de-nds-translation/blob/HEAD/translation_sentencepiece.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f6b944f50d3f15ae"}},{"code_sha256_prefix":"82404c105f013e2e","entry":"spm_args","repo":"mmcux/de-nds-translation","repo_kind":"listed","path":"translation_sentencepiece.py","file_url":"https://github.com/mmcux/de-nds-translation/blob/HEAD/translation_sentencepiece.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"82404c105f013e2e"}},{"code_sha256_prefix":"9c74fbbd91d14107","entry":"tokenize_nds","repo":"mmcux/de-nds-translation","repo_kind":"listed","path":"translation_model.py","file_url":"https://github.com/mmcux/de-nds-translation/blob/HEAD/translation_model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9c74fbbd91d14107"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}