{"url":"/dataset/oscar","name":"OSCAR","full_name":"OSCAR","description_markdown":"**OSCAR** or Open Super-large Crawled ALMAnaCH coRpus is a huge multilingual corpus obtained by language classification and filtering of the Common Crawl corpus using the goclassy architecture. The dataset used for training multilingual models such as BART incorporates 138 GB of text.","description_withheld":null,"homepage":"https://oscar-corpus.com/","introduced_date":"2020-06-11","introduced_date_note":null,"introduced_by":{"paper":"/paper/a-monolingual-approach-to-contextualized-word","title":"A Monolingual Approach to Contextualized Word Embeddings for Mid-Resource Languages","first_author":"Pedro Javier Ortiz Suárez","url":null},"license":{"name":"CC0 1.0","url":"https://creativecommons.org/publicdomain/zero/1.0/"},"modalities":[{"name":"Texts","url":"/datasets/modality/texts"}],"tasks":[{"name":"Language Modelling","url":"/task/language-modelling","datasets_with_task":"/datasets/task/language-modelling"}],"languages":[{"name":"English","url":"/datasets/language/english"},{"name":"French","url":"/datasets/language/french"},{"name":"Spanish","url":"/datasets/language/spanish"},{"name":"German","url":"/datasets/language/german"},{"name":"Italian","url":"/datasets/language/italian"},{"name":"Chinese","url":"/datasets/language/chinese"},{"name":"Bengali","url":"/datasets/language/bengali"},{"name":"Multilingual","url":"/datasets/language/multilingual"},{"name":"Japanese","url":"/datasets/language/japanese"},{"name":"Russian","url":"/datasets/language/russian"},{"name":"Portuguese","url":"/datasets/language/portuguese"},{"name":"Afrikaans","url":"/datasets/language/afrikaans"},{"name":"Albanian","url":"/datasets/language/albanian"},{"name":"Amharic","url":"/datasets/language/amharic"},{"name":"Arabic","url":"/datasets/language/arabic"},{"name":"Armenian","url":"/datasets/language/armenian"},{"name":"Basque","url":"/datasets/language/basque"},{"name":"Belarusian","url":"/datasets/language/belarusian"},{"name":"Breton","url":"/datasets/language/breton"},{"name":"Bulgarian","url":"/datasets/language/bulgarian"},{"name":"Catalan","url":"/datasets/language/catalan"},{"name":"Croatian","url":"/datasets/language/croatian"},{"name":"Czech","url":"/datasets/language/czech"},{"name":"Danish","url":"/datasets/language/danish"},{"name":"Dutch","url":"/datasets/language/dutch"},{"name":"Erzya","url":"/datasets/language/erzya"},{"name":"Estonian","url":"/datasets/language/estonian"},{"name":"Finnish","url":"/datasets/language/finnish"},{"name":"Galician","url":"/datasets/language/galician"},{"name":"Hebrew","url":"/datasets/language/hebrew"},{"name":"Hindi","url":"/datasets/language/hindi"},{"name":"Hungarian","url":"/datasets/language/hungarian"},{"name":"Icelandic","url":"/datasets/language/icelandic"},{"name":"Indonesian","url":"/datasets/language/indonesian"},{"name":"Irish","url":"/datasets/language/irish"},{"name":"Kazakh","url":"/datasets/language/kazakh"},{"name":"Korean","url":"/datasets/language/korean"},{"name":"Latin","url":"/datasets/language/latin"},{"name":"Latvian","url":"/datasets/language/latvian"},{"name":"Lithuanian","url":"/datasets/language/lithuanian"},{"name":"Maltese","url":"/datasets/language/maltese"},{"name":"Marathi","url":"/datasets/language/marathi"},{"name":"Modern Greek","url":"/datasets/language/modern-greek"},{"name":"Norwegian","url":"/datasets/language/norwegian"},{"name":"Persian","url":"/datasets/language/persian"},{"name":"Polish","url":"/datasets/language/polish"},{"name":"Romanian","url":"/datasets/language/romanian"},{"name":"Russia Buriat","url":"/datasets/language/russia-buriat"},{"name":"Sanskrit","url":"/datasets/language/sanskrit"},{"name":"Scottish Gaelic","url":"/datasets/language/scottish-gaelic"},{"name":"Serbian","url":"/datasets/language/serbian"},{"name":"Slovak","url":"/datasets/language/slovak"},{"name":"Slovenian","url":"/datasets/language/slovenian"},{"name":"Swedish","url":"/datasets/language/swedish"},{"name":"Tagalog","url":"/datasets/language/tagalog"},{"name":"Tamil","url":"/datasets/language/tamil"},{"name":"Telugu","url":"/datasets/language/telugu"},{"name":"Thai","url":"/datasets/language/thai"},{"name":"Turkish","url":"/datasets/language/turkish"},{"name":"Uighur","url":"/datasets/language/uighur"},{"name":"Ukrainian","url":"/datasets/language/ukrainian"},{"name":"Upper Sorbian","url":"/datasets/language/upper-sorbian"},{"name":"Urdu","url":"/datasets/language/urdu"},{"name":"Vietnamese","url":"/datasets/language/vietnamese"},{"name":"Welsh","url":"/datasets/language/welsh"},{"name":"Yoruba","url":"/datasets/language/yoruba"},{"name":"Yue Chinese","url":"/datasets/language/yue-chinese"},{"name":"Swahili","url":"/datasets/language/swahili"},{"name":"Aragonese","url":"/datasets/language/aragonese"},{"name":"Egyptian Arabic","url":"/datasets/language/egyptian-arabic"},{"name":"Assamese","url":"/datasets/language/assamese"},{"name":"Asturian","url":"/datasets/language/asturian"},{"name":"Avaric","url":"/datasets/language/avaric"},{"name":"South Azerbaijani","url":"/datasets/language/south-azerbaijani"},{"name":"Azerbaijani","url":"/datasets/language/azerbaijani"},{"name":"Bashkir","url":"/datasets/language/bashkir"},{"name":"Bavarian","url":"/datasets/language/bavarian"},{"name":"Central Bikol","url":"/datasets/language/central-bikol"},{"name":"Tibetan","url":"/datasets/language/tibetan"},{"name":"Bosnian","url":"/datasets/language/bosnian"},{"name":"Bishnupriya","url":"/datasets/language/bishnupriya"},{"name":"Chavacano","url":"/datasets/language/chavacano"},{"name":"Cebuano","url":"/datasets/language/cebuano"},{"name":"Chechen","url":"/datasets/language/chechen"},{"name":"Chuvash","url":"/datasets/language/chuvash"},{"name":"Central Kurdish","url":"/datasets/language/central-kurdish"},{"name":"Cornish","url":"/datasets/language/cornish"},{"name":"Dimli (individual language)","url":"/datasets/language/dimli-individual-language"},{"name":"Dhivehi","url":"/datasets/language/dhivehi"},{"name":"Lower Sorbian","url":"/datasets/language/lower-sorbian"},{"name":"Esperanto","url":"/datasets/language/esperanto"},{"name":"Northern Frisian","url":"/datasets/language/northern-frisian"},{"name":"Western Frisian","url":"/datasets/language/western-frisian"},{"name":"Goan Konkani","url":"/datasets/language/goan-konkani"},{"name":"Guarani","url":"/datasets/language/guarani"},{"name":"Gujarati","url":"/datasets/language/gujarati"},{"name":"Haitian","url":"/datasets/language/haitian"},{"name":"Serbo-Croatian","url":"/datasets/language/serbo-croatian"},{"name":"Ido","url":"/datasets/language/ido"},{"name":"Interlingue","url":"/datasets/language/interlingue"},{"name":"Iloko","url":"/datasets/language/iloko"},{"name":"Javanese","url":"/datasets/language/javanese"},{"name":"Lojban","url":"/datasets/language/lojban"},{"name":"Kannada","url":"/datasets/language/kannada"},{"name":"Georgian","url":"/datasets/language/georgian"},{"name":"Central Khmer","url":"/datasets/language/central-khmer"},{"name":"Kirghiz","url":"/datasets/language/kirghiz"},{"name":"Komi","url":"/datasets/language/komi"},{"name":"Karachay-Balkar","url":"/datasets/language/karachay-balkar"},{"name":"Kurdish","url":"/datasets/language/kurdish"},{"name":"Lao","url":"/datasets/language/lao"},{"name":"Lezghian","url":"/datasets/language/lezghian"},{"name":"Limburgan","url":"/datasets/language/limburgan"},{"name":"Lombard","url":"/datasets/language/lombard"},{"name":"Northern Luri","url":"/datasets/language/northern-luri"},{"name":"Luxembourgish","url":"/datasets/language/luxembourgish"},{"name":"Maithili","url":"/datasets/language/maithili"},{"name":"Malayalam","url":"/datasets/language/malayalam"},{"name":"Eastern Mari","url":"/datasets/language/eastern-mari"},{"name":"Minangkabau","url":"/datasets/language/minangkabau"},{"name":"Macedonian","url":"/datasets/language/macedonian"},{"name":"Malagasy","url":"/datasets/language/malagasy"},{"name":"Mongolian","url":"/datasets/language/mongolian"},{"name":"Western Mari","url":"/datasets/language/western-mari"},{"name":"Mirandese","url":"/datasets/language/mirandese"},{"name":"Burmese","url":"/datasets/language/burmese"},{"name":"Mazanderani","url":"/datasets/language/mazanderani"},{"name":"Neapolitan","url":"/datasets/language/neapolitan"},{"name":"Low German","url":"/datasets/language/low-german"},{"name":"Newari","url":"/datasets/language/newari"},{"name":"Norwegian Nynorsk","url":"/datasets/language/norwegian-nynorsk"},{"name":"Occitan (post 1500)","url":"/datasets/language/occitan-post-1500"},{"name":"Oriya (macrolanguage)","url":"/datasets/language/oriya-macrolanguage"},{"name":"Ossetian","url":"/datasets/language/ossetian"},{"name":"Pampanga","url":"/datasets/language/pampanga"},{"name":"Punjabi","url":"/datasets/language/punjabi"},{"name":"Piemontese","url":"/datasets/language/piemontese"},{"name":"Western Panjabi","url":"/datasets/language/western-panjabi"},{"name":"Pushto","url":"/datasets/language/pushto"},{"name":"Quechua","url":"/datasets/language/quechua"},{"name":"Romansh","url":"/datasets/language/romansh"},{"name":"Yakut","url":"/datasets/language/yakut"},{"name":"Sicilian","url":"/datasets/language/sicilian"},{"name":"Sinhala","url":"/datasets/language/sinhala"},{"name":"Sindhi","url":"/datasets/language/sindhi"},{"name":"Somali","url":"/datasets/language/somali"},{"name":"Sundanese","url":"/datasets/language/sundanese"},{"name":"Tatar","url":"/datasets/language/tatar"},{"name":"Tajik","url":"/datasets/language/tajik"},{"name":"Turkmen","url":"/datasets/language/turkmen"},{"name":"Tuvinian","url":"/datasets/language/tuvinian"},{"name":"Uzbek","url":"/datasets/language/uzbek"},{"name":"Venetian","url":"/datasets/language/venetian"},{"name":"Volapük","url":"/datasets/language/volap-k"},{"name":"Waray (Philippines)","url":"/datasets/language/waray-philippines"},{"name":"Walloon","url":"/datasets/language/walloon"},{"name":"Wu Chinese","url":"/datasets/language/wu-chinese"},{"name":"Kalmyk","url":"/datasets/language/kalmyk"},{"name":"Mingrelian","url":"/datasets/language/mingrelian"},{"name":"Yiddish","url":"/datasets/language/yiddish"},{"name":"Malay (individual language)","url":"/datasets/language/malay-individual-language"}],"variants":["OSCAR"],"data_loaders":[{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/oscar","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/oscar-corpus/OSCAR-2109","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/oscar-corpus/OSCAR-2201","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/nthngdy/oscar-small","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/munggok/KoPI-CC","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/munggok/KoPI-CC_News","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/munggok/KoPI","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/nthngdy/oscar-mini","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/acul3/KoPI-CC","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/acul3/KoPI-CC_News","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/acul3/KoPI","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/5w4n/OSCAR-2019-Burmese-fix","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/djstrong/oscar-small","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/oscar-corpus/OSCAR-2301","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/oscar-corpus/oscar-2301-hpc","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/oscar-corpus/colossal-oscar-1.0","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/oscar-corpus/oscar","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/Bahasalab/KoPI","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/oscar-corpus/community-oscar","frameworks":["tf","pytorch","jax"]},{"repo":"https://github.com/huggingface/datasets","url":"https://huggingface.co/datasets/mittagessen/oscar_subset","frameworks":["tf","pytorch","jax"]}],"num_papers_in_archive":64,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28"},"benchmarks":[],"papers_with_a_benchmark_row":[],"syntology_totals":{"read_at":"2026-09-24T18:15:14+00:00","papers_with_samples":0,"samples_harvested":0,"samples_ran":0,"samples_unverified":0,"pointer_only_for_licence":0,"papers_with_no_sample_that_ran":0,"note":"the per-paper counts above, summed; not a rate"},"papers_note":"The archive never published its papers-using-dataset list; these are papers with a leaderboard row on this dataset's benchmarks."}