{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/indonlu-benchmark-and-resources-for","title":"IndoNLU: Benchmark and Resources for Evaluating Indonesian Natural Language Understanding","arxiv_id":"2009.05387","date":"2020-09-11","proceeding":"Asian Chapter of the Association for Computational Linguistics 2020","authors":["Bryan Wilie","Karissa Vincentio","Genta Indra Winata","Samuel Cahyawijaya","Xiaohong Li","Zhi Yuan Lim","Sidik Soleman","Rahmad Mahendra","Pascale Fung","Syafri Bahar","Ayu Purwarianti"],"abstract":"Although Indonesian is known to be the fourth most frequently used language over the internet, the research progress on this language in the natural language processing (NLP) is slow-moving due to a lack of available resources. In response, we introduce the first-ever vast resource for the training, evaluating, and benchmarking on Indonesian natural language understanding (IndoNLU) tasks. IndoNLU includes twelve tasks, ranging from single sentence classification to pair-sentences sequence labeling with different levels of complexity. The datasets for the tasks lie in different domains and styles to ensure task diversity. We also provide a set of Indonesian pre-trained models (IndoBERT) trained from a large and clean Indonesian dataset Indo4B collected from publicly available sources such as social media texts, blogs, news, and websites. We release baseline models for all twelve tasks, as well as the framework for benchmark evaluation, and thus it enables everyone to benchmark their system performances.","url_abs":"https://arxiv.org/abs/2009.05387v3","url_pdf":"https://arxiv.org/pdf/2009.05387v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"indonlu-benchmark-and-resources-for","repo_url":"https://github.com/indobenchmark/indonlu","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"indonlu-benchmark-and-resources-for","repo_url":"https://github.com/LazarusNLP/NusaBERT","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"indonlu-benchmark-and-resources-for","repo_url":"https://github.com/assulthoni/TweetSentimentIndoBERT","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"GPL-2.0"}}],"tasks":[{"task_slug":"benchmarking","task_name":"Benchmarking"},{"task_slug":"diversity","task_name":"Diversity"},{"task_slug":"natural-language-understanding","task_name":"Natural Language Understanding"},{"task_slug":"sentence","task_name":"Sentence"},{"task_slug":"sentence-classification","task_name":"Sentence Classification"}],"methods":[],"datasets_introduced":[{"slug":"indonlu-benchmark","name":"IndoNLU Benchmark","full_name":""}],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2009.05387","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2009.05387"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/indobenchmark/indonlu","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/LazarusNLP/NusaBERT","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/assulthoni/TweetSentimentIndoBERT","reach":{"status":"ok","spdx":"GPL-2.0"}}],"summary":{"ran_honours":1,"ran_draft_wrong":1,"unverified":12},"by_repo_kind":{"official":{"samples":14,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"e9c7d263fd12f86d","entry":"get_lr","repo":"indobenchmark/indonlu","repo_kind":"official","path":"predict.py","file_url":"https://github.com/indobenchmark/indonlu/blob/HEAD/predict.py","link_basis":"plan_row","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e9c7d263fd12f86d"}},{"code_sha256_prefix":"2c8f98e551054b67","entry":"metrics_to_string","repo":"indobenchmark/indonlu","repo_kind":"official","path":"predict.py","file_url":"https://github.com/indobenchmark/indonlu/blob/HEAD/predict.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2c8f98e551054b67"}},{"code_sha256_prefix":"0fb5de45230b147d","entry":"aspect_extraction_metrics_fn","repo":"indobenchmark/indonlu","repo_kind":"official","path":"utils/metrics.py","file_url":"https://github.com/indobenchmark/indonlu/blob/HEAD/utils/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0fb5de45230b147d"}},{"code_sha256_prefix":"536eda93d03af310","entry":"emotion_detection_metrics_fn","repo":"indobenchmark/indonlu","repo_kind":"official","path":"utils/metrics.py","file_url":"https://github.com/indobenchmark/indonlu/blob/HEAD/utils/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"536eda93d03af310"}},{"code_sha256_prefix":"abc247eaad47c595","entry":"end_of_chunk","repo":"indobenchmark/indonlu","repo_kind":"official","path":"utils/conlleval.py","file_url":"https://github.com/indobenchmark/indonlu/blob/HEAD/utils/conlleval.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"abc247eaad47c595"}},{"code_sha256_prefix":"b0608daef236473c","entry":"forward_sequence_classification","repo":"indobenchmark/indonlu","repo_kind":"official","path":"utils/forward_fn.py","file_url":"https://github.com/indobenchmark/indonlu/blob/HEAD/utils/forward_fn.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b0608daef236473c"}},{"code_sha256_prefix":"53f0892c0d642652","entry":"forward_sequence_multi_classification","repo":"indobenchmark/indonlu","repo_kind":"official","path":"utils/forward_fn.py","file_url":"https://github.com/indobenchmark/indonlu/blob/HEAD/utils/forward_fn.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"53f0892c0d642652"}},{"code_sha256_prefix":"0e01586952d91f1a","entry":"forward_word_classification","repo":"indobenchmark/indonlu","repo_kind":"official","path":"utils/forward_fn.py","file_url":"https://github.com/indobenchmark/indonlu/blob/HEAD/utils/forward_fn.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0e01586952d91f1a"}},{"code_sha256_prefix":"931c15ea00fd6e21","entry":"gen_embeddings","repo":"indobenchmark/indonlu","repo_kind":"official","path":"utils/functions.py","file_url":"https://github.com/indobenchmark/indonlu/blob/HEAD/utils/functions.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"931c15ea00fd6e21"}},{"code_sha256_prefix":"4384e3778f7c4a96","entry":"load_vocab","repo":"indobenchmark/indonlu","repo_kind":"official","path":"utils/functions.py","file_url":"https://github.com/indobenchmark/indonlu/blob/HEAD/utils/functions.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4384e3778f7c4a96"}},{"code_sha256_prefix":"b9229e6df7796a72","entry":"ner_metrics_fn","repo":"indobenchmark/indonlu","repo_kind":"official","path":"utils/metrics.py","file_url":"https://github.com/indobenchmark/indonlu/blob/HEAD/utils/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b9229e6df7796a72"}},{"code_sha256_prefix":"2e9480de024f3eec","entry":"parse_tag","repo":"indobenchmark/indonlu","repo_kind":"official","path":"utils/conlleval.py","file_url":"https://github.com/indobenchmark/indonlu/blob/HEAD/utils/conlleval.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2e9480de024f3eec"}},{"code_sha256_prefix":"da384f25986bd37e","entry":"predict","repo":"indobenchmark/indonlu","repo_kind":"official","path":"predict.py","file_url":"https://github.com/indobenchmark/indonlu/blob/HEAD/predict.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"da384f25986bd37e"}},{"code_sha256_prefix":"8a646b990ee217c9","entry":"start_of_chunk","repo":"indobenchmark/indonlu","repo_kind":"official","path":"utils/conlleval.py","file_url":"https://github.com/indobenchmark/indonlu/blob/HEAD/utils/conlleval.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8a646b990ee217c9"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}