{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/kalahi-a-handcrafted-grassroots-cultural-llm","title":"Kalahi: A handcrafted, grassroots cultural LLM evaluation suite for Filipino","arxiv_id":"2409.15380","date":"2024-09-20","proceeding":null,"authors":["Jann Railey Montalan","Jian Gang Ngui","Wei Qi Leong","Yosephine Susanto","Hamsawardhini Rengarajan","Alham Fikri Aji","William Chandra Tjhi"],"abstract":"Multilingual large language models (LLMs) today may not necessarily provide culturally appropriate and relevant responses to its Filipino users. We introduce Kalahi, a cultural LLM evaluation suite collaboratively created by native Filipino speakers. It is composed of 150 high-quality, handcrafted and nuanced prompts that test LLMs for generations that are relevant to shared Filipino cultural knowledge and values. Strong LLM performance in Kalahi indicates a model's ability to generate responses similar to what an average Filipino would say or do in a given situation. We conducted experiments on LLMs with multilingual and Filipino language support. Results show that Kalahi, while trivial for Filipinos, is challenging for LLMs, with the best model answering only 46.0% of the questions correctly compared to native Filipino performance of 89.10%. Thus, Kalahi can be used to accurately and reliably evaluate Filipino cultural representation in LLMs.","url_abs":"https://arxiv.org/abs/2409.15380v3","url_pdf":"https://arxiv.org/pdf/2409.15380v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"kalahi-a-handcrafted-grassroots-cultural-llm","repo_url":"https://github.com/aisingapore/kalahi","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"CC-BY-4.0"}}],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2409.15380","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2409.15380"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/aisingapore/kalahi","reach":{"status":"ok","spdx":"CC-BY-4.0"}}],"summary":{"ran":3},"by_repo_kind":{"official":{"samples":3,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":3,"samples":[{"code_sha256_prefix":"b89a6d986d96e4fa","entry":"format_prompt","repo":"aisingapore/kalahi","repo_kind":"official","path":"kalahi/utilities.py","file_url":"https://github.com/aisingapore/kalahi/blob/HEAD/kalahi/utilities.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"CC-BY-4.0","inline_ok":false,"mcp_get_code":{"code_sha256":"b89a6d986d96e4fa"}},{"code_sha256_prefix":"f1eb5a7d7cd86098","entry":"format_prompt_with_reference","repo":"aisingapore/kalahi","repo_kind":"official","path":"kalahi/utilities.py","file_url":"https://github.com/aisingapore/kalahi/blob/HEAD/kalahi/utilities.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"CC-BY-4.0","inline_ok":false,"mcp_get_code":{"code_sha256":"f1eb5a7d7cd86098"}},{"code_sha256_prefix":"a018415e35b1d137","entry":"load_prompts","repo":"aisingapore/kalahi","repo_kind":"official","path":"kalahi/utilities.py","file_url":"https://github.com/aisingapore/kalahi/blob/HEAD/kalahi/utilities.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"CC-BY-4.0","inline_ok":false,"mcp_get_code":{"code_sha256":"a018415e35b1d137"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}