{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/improving-hateful-memes-detection-via","title":"Improving Hateful Meme Detection through Retrieval-Guided Contrastive Learning","arxiv_id":"2311.08110","date":"2023-11-14","proceeding":null,"authors":["Jingbiao Mei","Jinghong Chen","Weizhe Lin","Bill Byrne","Marcus Tomalin"],"abstract":"Hateful memes have emerged as a significant concern on the Internet. Detecting hateful memes requires the system to jointly understand the visual and textual modalities. Our investigation reveals that the embedding space of existing CLIP-based systems lacks sensitivity to subtle differences in memes that are vital for correct hatefulness classification. We propose constructing a hatefulness-aware embedding space through retrieval-guided contrastive training. Our approach achieves state-of-the-art performance on the HatefulMemes dataset with an AUROC of 87.0, outperforming much larger fine-tuned large multimodal models. We demonstrate a retrieval-based hateful memes detection system, which is capable of identifying hatefulness based on data unseen in training. This allows developers to update the hateful memes detection system by simply adding new examples without retraining, a desirable feature for real services in the constantly evolving landscape of hateful memes on the Internet.","url_abs":"https://arxiv.org/abs/2311.08110v3","url_pdf":"https://arxiv.org/pdf/2311.08110v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"improving-hateful-memes-detection-via","repo_url":"https://github.com/JingbiaoMei/RGCL","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"contrastive-learning","task_name":"Contrastive Learning"},{"task_slug":"hateful-meme-classification","task_name":"Hateful Meme Classification"},{"task_slug":"meme-classification","task_name":"Meme Classification"},{"task_slug":"retrieval","task_name":"Retrieval"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/hateful-meme-classification-on-harmeme","task":"Hateful Meme Classification","dataset":"HarMeme","model":"RGCL","rank_in_archive_order":5,"of":8,"metrics":{"AUROC":"91.80","Accuracy":"87.00"},"uses_additional_data":false},{"leaderboard":"/sota/hateful-meme-classification-on-harm-p","task":"Hateful Meme Classification","dataset":"Harm-P","model":"RGCL","rank_in_archive_order":3,"of":5,"metrics":{"Accuracy":"89.9","F1":"89.5"},"uses_additional_data":false},{"leaderboard":"/sota/hateful-meme-classification-on-pridemm","task":"Hateful Meme Classification","dataset":"PrideMM","model":"RGCL","rank_in_archive_order":3,"of":7,"metrics":{"Accuracy":"76.3","F1":"76.5"},"uses_additional_data":false},{"leaderboard":"/sota/meme-classification-on-hateful-memes","task":"Meme Classification","dataset":"Hateful Memes","model":"RGCL (CLIP)","rank_in_archive_order":5,"of":17,"metrics":{"Accuracy":"0.788","ROC-AUC":"0.870"},"uses_additional_data":false},{"leaderboard":"/sota/meme-classification-on-multioff","task":"Meme Classification","dataset":"MultiOFF","model":"RGCL","rank_in_archive_order":2,"of":5,"metrics":{"Accuracy":"67.1","F1":"58.1"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2311.08110","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.08110"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/JingbiaoMei/RGCL","reach":null}],"summary":{"ran_draft_wrong":1},"by_repo_kind":{"official":{"samples":1,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"41bfd5b116fe8740","entry":"parse_args_sys","repo":"JingbiaoMei/RGCL","repo_kind":"official","path":"src/utils/generate_CLIP_embedding_HF.py","file_url":"https://github.com/JingbiaoMei/RGCL/blob/HEAD/src/utils/generate_CLIP_embedding_HF.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"41bfd5b116fe8740"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}