{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/can-llm-generate-culturally-relevant","title":"Can LLM Generate Culturally Relevant Commonsense QA Data? Case Study in Indonesian and Sundanese","arxiv_id":"2402.17302","date":"2024-02-27","proceeding":null,"authors":["Rifki Afina Putri","Faiz Ghifari Haznitrama","Dea Adhista","Alice Oh"],"abstract":"Large Language Models (LLMs) are increasingly being used to generate synthetic data for training and evaluating models. However, it is unclear whether they can generate a good quality of question answering (QA) dataset that incorporates knowledge and cultural nuance embedded in a language, especially for low-resource languages. In this study, we investigate the effectiveness of using LLMs in generating culturally relevant commonsense QA datasets for Indonesian and Sundanese languages. To do so, we create datasets for these languages using various methods involving both LLMs and human annotators, resulting in ~4.5K questions per language (~9K in total), making our dataset the largest of its kind. Our experiments show that automatic data adaptation from an existing English dataset is less effective for Sundanese. Interestingly, using the direct generation method on the target language, GPT-4 Turbo can generate questions with adequate general knowledge in both languages, albeit not as culturally 'deep' as humans. We also observe a higher occurrence of fluency errors in the Sundanese dataset, highlighting the discrepancy between medium- and lower-resource languages.","url_abs":"https://arxiv.org/abs/2402.17302v3","url_pdf":"https://arxiv.org/pdf/2402.17302v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"can-llm-generate-culturally-relevant","repo_url":"https://github.com/rifkiaputri/id-csqa","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}}],"tasks":[{"task_slug":"general-knowledge","task_name":"General Knowledge"},{"task_slug":"question-answering","task_name":"Question Answering"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt-4","method_name":"GPT-4"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2402.17302","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2402.17302"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/rifkiaputri/id-csqa","reach":{"status":"ok"}}],"summary":{"ran":8},"by_repo_kind":{"official":{"samples":8,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":8,"samples":[{"code_sha256_prefix":"0ad74b0ae3b4f7a3","entry":"clean_data","repo":"rifkiaputri/id-csqa","repo_kind":"official","path":"generation/rephrase.py","file_url":"https://github.com/rifkiaputri/id-csqa/blob/HEAD/generation/rephrase.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0ad74b0ae3b4f7a3"}},{"code_sha256_prefix":"a67bac06aac78ad4","entry":"clean_response_keys","repo":"rifkiaputri/id-csqa","repo_kind":"official","path":"generation/rephrase.py","file_url":"https://github.com/rifkiaputri/id-csqa/blob/HEAD/generation/rephrase.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a67bac06aac78ad4"}},{"code_sha256_prefix":"08d54b12b8e5a618","entry":"divide_list_into_chunks","repo":"rifkiaputri/id-csqa","repo_kind":"official","path":"utils/helpers.py","file_url":"https://github.com/rifkiaputri/id-csqa/blob/HEAD/utils/helpers.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"08d54b12b8e5a618"}},{"code_sha256_prefix":"5a0d7504786a34f4","entry":"filter_concept_strict","repo":"rifkiaputri/id-csqa","repo_kind":"official","path":"generation/filter.py","file_url":"https://github.com/rifkiaputri/id-csqa/blob/HEAD/generation/filter.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5a0d7504786a34f4"}},{"code_sha256_prefix":"4087fe280199d65b","entry":"load_csv_data","repo":"rifkiaputri/id-csqa","repo_kind":"official","path":"utils/helpers.py","file_url":"https://github.com/rifkiaputri/id-csqa/blob/HEAD/utils/helpers.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4087fe280199d65b"}},{"code_sha256_prefix":"e43ae926d648c185","entry":"postprocess_data","repo":"rifkiaputri/id-csqa","repo_kind":"official","path":"generation/generate_v3.py","file_url":"https://github.com/rifkiaputri/id-csqa/blob/HEAD/generation/generate_v3.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e43ae926d648c185"}},{"code_sha256_prefix":"0802f23aa02a7672","entry":"postprocess_result_all","repo":"rifkiaputri/id-csqa","repo_kind":"official","path":"generation/rephrase.py","file_url":"https://github.com/rifkiaputri/id-csqa/blob/HEAD/generation/rephrase.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0802f23aa02a7672"}},{"code_sha256_prefix":"fbdd61a4a2a9163d","entry":"remove_unused_keys","repo":"rifkiaputri/id-csqa","repo_kind":"official","path":"utils/helpers.py","file_url":"https://github.com/rifkiaputri/id-csqa/blob/HEAD/utils/helpers.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fbdd61a4a2a9163d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}