{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/xtreme-up-a-user-centric-scarce-data","title":"XTREME-UP: A User-Centric Scarce-Data Benchmark for Under-Represented Languages","arxiv_id":"2305.11938","date":"2023-05-19","proceeding":null,"authors":["Sebastian Ruder","Jonathan H. Clark","Alexander Gutkin","Mihir Kale","Min Ma","Massimo Nicosia","Shruti Rijhwani","Parker Riley","Jean-Michel A. Sarr","Xinyi Wang","John Wieting","Nitish Gupta","Anna Katanova","Christo Kirov","Dana L. Dickinson","Brian Roark","Bidisha Samanta","Connie Tao","David I. Adelani","Vera Axelrod","Isaac Caswell","Colin Cherry","Dan Garrette","Reeve Ingle","Melvin Johnson","Dmitry Panteleev","Partha Talukdar"],"abstract":"Data scarcity is a crucial issue for the development of highly multilingual NLP systems. Yet for many under-represented languages (ULs) -- languages for which NLP re-search is particularly far behind in meeting user needs -- it is feasible to annotate small amounts of data. Motivated by this, we propose XTREME-UP, a benchmark defined by: its focus on the scarce-data scenario rather than zero-shot; its focus on user-centric tasks -- tasks with broad adoption by speakers of high-resource languages; and its focus on under-represented languages where this scarce-data scenario tends to be most realistic. XTREME-UP evaluates the capabilities of language models across 88 under-represented languages over 9 key user-centric technologies including ASR, OCR, MT, and information access tasks that are of general utility. We create new datasets for OCR, autocomplete, semantic parsing, and transliteration, and build on and refine existing datasets for other tasks. XTREME-UP provides methodology for evaluating many modeling scenarios including text-only, multi-modal (vision, audio, and text),supervised parameter tuning, and in-context learning. We evaluate commonly used models on the benchmark. We release all code and scripts to train and evaluate models","url_abs":"https://arxiv.org/abs/2305.11938v2","url_pdf":"https://arxiv.org/pdf/2305.11938v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"xtreme-up-a-user-centric-scarce-data","repo_url":"https://github.com/google-research/xtreme-up","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"jax","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"in-context-learning","task_name":"In-Context Learning"},{"task_slug":"multilingual-nlp","task_name":"Multilingual NLP"},{"task_slug":"optical-character-recognition","task_name":"Optical Character Recognition (OCR)"},{"task_slug":"semantic-parsing","task_name":"Semantic Parsing"},{"task_slug":"transliteration","task_name":"Transliteration"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2305.11938","atlas_url":"https://app.syntology.ai/?focus=2305.11938","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.11938"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/google-research/xtreme-up","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"unverified":11},"by_repo_kind":{"official":{"samples":11,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"9b8db034cb6fc339","entry":"average","repo":"google-research/xtreme-up","repo_kind":"official","path":"evaluation/score_system.py","file_url":"https://github.com/google-research/xtreme-up/blob/HEAD/evaluation/score_system.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9b8db034cb6fc339"}},{"code_sha256_prefix":"ffe1eb8489531d61","entry":"compute_similarities","repo":"google-research/xtreme-up","repo_kind":"official","path":"baseline/compute_nearest_neighbors.py","file_url":"https://github.com/google-research/xtreme-up/blob/HEAD/baseline/compute_nearest_neighbors.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ffe1eb8489531d61"}},{"code_sha256_prefix":"eda34985ba4beae0","entry":"dummy_metric","repo":"google-research/xtreme-up","repo_kind":"official","path":"baseline/autocomplete_tasks.py","file_url":"https://github.com/google-research/xtreme-up/blob/HEAD/baseline/autocomplete_tasks.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"eda34985ba4beae0"}},{"code_sha256_prefix":"abd225a4df0fa702","entry":"gather_predictions","repo":"google-research/xtreme-up","repo_kind":"official","path":"baseline/autocomplete_tasks.py","file_url":"https://github.com/google-research/xtreme-up/blob/HEAD/baseline/autocomplete_tasks.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"abd225a4df0fa702"}},{"code_sha256_prefix":"dd148fcf0cf05289","entry":"get_task_info","repo":"google-research/xtreme-up","repo_kind":"official","path":"baseline/jsonl_to_seqio_tsv_lib.py","file_url":"https://github.com/google-research/xtreme-up/blob/HEAD/baseline/jsonl_to_seqio_tsv_lib.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"dd148fcf0cf05289"}},{"code_sha256_prefix":"086e5dd32d79621b","entry":"l2_normalize","repo":"google-research/xtreme-up","repo_kind":"official","path":"baseline/compute_nearest_neighbors.py","file_url":"https://github.com/google-research/xtreme-up/blob/HEAD/baseline/compute_nearest_neighbors.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"086e5dd32d79621b"}},{"code_sha256_prefix":"7a4a03eb0fe8c1df","entry":"lowercase","repo":"google-research/xtreme-up","repo_kind":"official","path":"baseline/retrieval_tasks.py","file_url":"https://github.com/google-research/xtreme-up/blob/HEAD/baseline/retrieval_tasks.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7a4a03eb0fe8c1df"}},{"code_sha256_prefix":"cb0b499a715ebd32","entry":"preprocess_autocomplete_example","repo":"google-research/xtreme-up","repo_kind":"official","path":"baseline/jsonl_to_seqio_tsv_lib.py","file_url":"https://github.com/google-research/xtreme-up/blob/HEAD/baseline/jsonl_to_seqio_tsv_lib.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cb0b499a715ebd32"}},{"code_sha256_prefix":"11b74ed4666eca82","entry":"process_example","repo":"google-research/xtreme-up","repo_kind":"official","path":"baseline/jsonl_to_seqio_tsv_lib.py","file_url":"https://github.com/google-research/xtreme-up/blob/HEAD/baseline/jsonl_to_seqio_tsv_lib.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"11b74ed4666eca82"}},{"code_sha256_prefix":"16cbaba2dd46e92f","entry":"split_and_rekey_processor","repo":"google-research/xtreme-up","repo_kind":"official","path":"baseline/retrieval_tasks.py","file_url":"https://github.com/google-research/xtreme-up/blob/HEAD/baseline/retrieval_tasks.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"16cbaba2dd46e92f"}},{"code_sha256_prefix":"f89c4c37abd368b2","entry":"to_inference_pair","repo":"google-research/xtreme-up","repo_kind":"official","path":"baseline/retrieval_tasks.py","file_url":"https://github.com/google-research/xtreme-up/blob/HEAD/baseline/retrieval_tasks.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f89c4c37abd368b2"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}