{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/multilingual-open-text-1-0-public-domain-news","title":"Multilingual Open Text Release 1: Public Domain News in 44 Languages","arxiv_id":"2201.05609","date":"2022-01-14","proceeding":"LREC 2022 6","authors":["Chester Palen-Michel","June Kim","Constantine Lignos"],"abstract":"We present Multilingual Open Text (MOT), a new multilingual corpus containing text in 44 languages, many of which have limited existing text resources for natural language processing. The first release of the corpus contains over 2.8 million news articles and an additional 1 million short snippets (photo captions, video descriptions, etc.) published between 2001--2022 and collected from Voice of America's news websites. We describe our process for collecting, filtering, and processing the data. The source material is in the public domain, our collection is licensed using a creative commons license (CC BY 4.0), and all software used to create the corpus is released under the MIT License. The corpus will be regularly updated as additional documents are published.","url_abs":"https://arxiv.org/abs/2201.05609v2","url_pdf":"https://arxiv.org/pdf/2201.05609v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"multilingual-open-text-1-0-public-domain-news","repo_url":"https://github.com/VietHoang1710/khmer-nltk","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"multilingual-open-text-1-0-public-domain-news","repo_url":"https://github.com/bltlab/mot","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"multilingual-open-text-1-0-public-domain-news","repo_url":"https://github.com/VietHoang1512/khmer-nltk","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"articles","task_name":"Articles"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2201.05609","atlas_url":"https://app.syntology.ai/?focus=2201.05609","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2201.05609"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/VietHoang1512/khmer-nltk","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/bltlab/mot","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/VietHoang1710/khmer-nltk","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":13,"unverified":1},"by_repo_kind":{"official":{"samples":13,"ran":12,"repositories":1},"listed":{"samples":1,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"ac259c52c934e834","entry":"author_allowed","repo":"bltlab/mot","repo_kind":"official","path":"extraction/dump_documents.py","file_url":"https://github.com/bltlab/mot/blob/HEAD/extraction/dump_documents.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ac259c52c934e834"}},{"code_sha256_prefix":"6bd8e8b915fc59a5","entry":"average_chars","repo":"bltlab/mot","repo_kind":"official","path":"qualitychecks/character_average.py","file_url":"https://github.com/bltlab/mot/blob/HEAD/qualitychecks/character_average.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6bd8e8b915fc59a5"}},{"code_sha256_prefix":"1cf40a8bef19a322","entry":"clean_fields","repo":"bltlab/mot","repo_kind":"official","path":"extraction/downloadsitemaps.py","file_url":"https://github.com/bltlab/mot/blob/HEAD/extraction/downloadsitemaps.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1cf40a8bef19a322"}},{"code_sha256_prefix":"3c6c78775c3fb35c","entry":"confident_multiple_languages","repo":"bltlab/mot","repo_kind":"official","path":"extraction/extracttext.py","file_url":"https://github.com/bltlab/mot/blob/HEAD/extraction/extracttext.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3c6c78775c3fb35c"}},{"code_sha256_prefix":"0dcddfcbdeb0e1b8","entry":"count_chars","repo":"bltlab/mot","repo_kind":"official","path":"qualitychecks/character_count.py","file_url":"https://github.com/bltlab/mot/blob/HEAD/qualitychecks/character_count.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0dcddfcbdeb0e1b8"}},{"code_sha256_prefix":"cb050e89f262217f","entry":"filter_confident_langs","repo":"bltlab/mot","repo_kind":"official","path":"extraction/extracttext.py","file_url":"https://github.com/bltlab/mot/blob/HEAD/extraction/extracttext.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cb050e89f262217f"}},{"code_sha256_prefix":"83d45f7a39a14210","entry":"get_publication_date_from_utag","repo":"bltlab/mot","repo_kind":"official","path":"extraction/utils.py","file_url":"https://github.com/bltlab/mot/blob/HEAD/extraction/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"83d45f7a39a14210"}},{"code_sha256_prefix":"8905f3eb7b69542e","entry":"gunzip","repo":"bltlab/mot","repo_kind":"official","path":"extraction/downloadsitemaps.py","file_url":"https://github.com/bltlab/mot/blob/HEAD/extraction/downloadsitemaps.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8905f3eb7b69542e"}},{"code_sha256_prefix":"7916cce5b903f3c4","entry":"is_valid","repo":"bltlab/mot","repo_kind":"official","path":"extraction/scraper.py","file_url":"https://github.com/bltlab/mot/blob/HEAD/extraction/scraper.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7916cce5b903f3c4"}},{"code_sha256_prefix":"365063f7b2778eda","entry":"post_process","repo":"VietHoang1512/khmer-nltk","repo_kind":"listed","path":"khmernltk/utils/data.py","file_url":"https://github.com/VietHoang1512/khmer-nltk/blob/HEAD/khmernltk/utils/data.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"365063f7b2778eda"}},{"code_sha256_prefix":"0df7e49ac86de638","entry":"predict_language","repo":"bltlab/mot","repo_kind":"official","path":"extraction/extracttext.py","file_url":"https://github.com/bltlab/mot/blob/HEAD/extraction/extracttext.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0df7e49ac86de638"}},{"code_sha256_prefix":"6b3a7b351209d121","entry":"read_voa_domains","repo":"bltlab/mot","repo_kind":"official","path":"extraction/downloadsitemaps.py","file_url":"https://github.com/bltlab/mot/blob/HEAD/extraction/downloadsitemaps.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6b3a7b351209d121"}},{"code_sha256_prefix":"911b9a345e873933","entry":"stringify_mongovalues","repo":"bltlab/mot","repo_kind":"official","path":"extraction/dump_documents.py","file_url":"https://github.com/bltlab/mot/blob/HEAD/extraction/dump_documents.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"911b9a345e873933"}},{"code_sha256_prefix":"4adb3f5772a9e273","entry":"doc_to_dict","repo":"bltlab/mot","repo_kind":"official","path":"extraction/dump_documents.py","file_url":"https://github.com/bltlab/mot/blob/HEAD/extraction/dump_documents.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4adb3f5772a9e273"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}