{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/crehate-cross-cultural-re-annotation-of","title":"Exploring Cross-Cultural Differences in English Hate Speech Annotations: From Dataset Construction to Analysis","arxiv_id":"2308.16705","date":"2023-08-31","proceeding":null,"authors":["Nayeon Lee","Chani Jung","Junho Myung","Jiho Jin","Jose Camacho-Collados","Juho Kim","Alice Oh"],"abstract":"Warning: this paper contains content that may be offensive or upsetting. Most hate speech datasets neglect the cultural diversity within a single language, resulting in a critical shortcoming in hate speech detection. To address this, we introduce CREHate, a CRoss-cultural English Hate speech dataset. To construct CREHate, we follow a two-step procedure: 1) cultural post collection and 2) cross-cultural annotation. We sample posts from the SBIC dataset, which predominantly represents North America, and collect posts from four geographically diverse English-speaking countries (Australia, United Kingdom, Singapore, and South Africa) using culturally hateful keywords we retrieve from our survey. Annotations are collected from the four countries plus the United States to establish representative labels for each country. Our analysis highlights statistically significant disparities across countries in hate speech annotations. Only 56.2% of the posts in CREHate achieve consensus among all countries, with the highest pairwise label difference rate of 26%. Qualitative analysis shows that label disagreement occurs mostly due to different interpretations of sarcasm and the personal bias of annotators on divisive topics. Lastly, we evaluate large language models (LLMs) under a zero-shot setting and show that current LLMs tend to show higher accuracies on Anglosphere country labels in CREHate. Our dataset and codes are available at: https://github.com/nlee0212/CREHate","url_abs":"https://arxiv.org/abs/2308.16705v3","url_pdf":"https://arxiv.org/pdf/2308.16705v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"crehate-cross-cultural-re-annotation-of","repo_url":"https://github.com/nlee0212/crehate","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"hate-speech-detection","task_name":"Hate Speech Detection"},{"task_slug":"transfer-learning","task_name":"Transfer Learning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2308.16705","atlas_url":"https://app.syntology.ai/?focus=2308.16705","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2308.16705"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/nlee0212/crehate","reach":{"status":"ok"}}],"summary":{"ran_draft_wrong":3,"unverified":1},"by_repo_kind":{"official":{"samples":4,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":4,"samples":[{"code_sha256_prefix":"c083b5f35fb91864","entry":"make_prompt","repo":"nlee0212/crehate","repo_kind":"official","path":"llm_inference.py","file_url":"https://github.com/nlee0212/crehate/blob/HEAD/llm_inference.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c083b5f35fb91864"}},{"code_sha256_prefix":"b7538ae6bdebdf32","entry":"raw2prediction","repo":"nlee0212/crehate","repo_kind":"official","path":"llm_inference.py","file_url":"https://github.com/nlee0212/crehate/blob/HEAD/llm_inference.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b7538ae6bdebdf32"}},{"code_sha256_prefix":"0a9e6c3ce2aa57e1","entry":"test","repo":"nlee0212/crehate","repo_kind":"official","path":"finetune/finetune.py","file_url":"https://github.com/nlee0212/crehate/blob/HEAD/finetune/finetune.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0a9e6c3ce2aa57e1"}},{"code_sha256_prefix":"21935adb83122f90","entry":"prediction_2_label","repo":"nlee0212/crehate","repo_kind":"official","path":"llm_inference.py","file_url":"https://github.com/nlee0212/crehate/blob/HEAD/llm_inference.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"21935adb83122f90"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}