{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/a-benchmark-for-toxic-comment-classification","title":"A benchmark for toxic comment classification on Civil Comments dataset","arxiv_id":"2301.11125","date":"2023-01-26","proceeding":null,"authors":["Corentin Duchene","Henri Jamet","Pierre Guillaume","Reda Dehak"],"abstract":"Toxic comment detection on social media has proven to be essential for content moderation. This paper compares a wide set of different models on a highly skewed multi-label hate speech dataset. We consider inference time and several metrics to measure performance and bias in our comparison. We show that all BERTs have similar performance regardless of the size, optimizations or language used to pre-train the models. RNNs are much faster at inference than any of the BERT. BiLSTM remains a good compromise between performance and inference time. RoBERTa with Focal Loss offers the best performance on biases and AUROC. However, DistilBERT combines both good AUROC and a low inference time. All models are affected by the bias of associating identities. BERT, RNN, and XLNet are less sensitive than the CNN and Compact Convolutional Transformers.","url_abs":"https://arxiv.org/abs/2301.11125v1","url_pdf":"https://arxiv.org/pdf/2301.11125v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"a-benchmark-for-toxic-comment-classification","repo_url":"https://github.com/Nigiva/hatespeech-detection-models","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"toxic-comment-classification","task_name":"Toxic Comment Classification"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"bilstm","method_name":"BiLSTM"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"distilbert","method_name":"DistilBERT"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"focal-loss","method_name":"Focal Loss"},{"method_slug":"lstm","method_name":"LSTM"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"roberta","method_name":"RoBERTa"},{"method_slug":"sentencepiece","method_name":"SentencePiece"},{"method_slug":"sigmoid-activation","method_name":"Sigmoid Activation"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"tanh-activation","method_name":"Tanh Activation"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"},{"method_slug":"xlnet","method_name":"XLNet"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/toxic-comment-classification-on-civil","task":"Toxic Comment Classification","dataset":"Civil Comments","model":"RoBERTa Focal Loss","rank_in_archive_order":1,"of":22,"metrics":{"AUROC":"0.9818","GMB BNSP":"0.9581","GMB BPSN":"0.901","GMB Subgroup":"0.8807","Macro F1":"0.4648","Micro F1":"0.5524","Precision":"0.4017","Recall":"0.8839"},"uses_additional_data":false},{"leaderboard":"/sota/toxic-comment-classification-on-civil","task":"Toxic Comment Classification","dataset":"Civil Comments","model":"AlBERT","rank_in_archive_order":2,"of":22,"metrics":{"AUROC":"0.979","GMB BNSP":"0.9499","GMB BPSN":"0.8982","GMB Subgroup":"0.8734","Macro F1":"0.3541","Micro F1":"0.4845","Precision":"0.3247","Recall":"0.9104"},"uses_additional_data":false},{"leaderboard":"/sota/toxic-comment-classification-on-civil","task":"Toxic Comment Classification","dataset":"Civil Comments","model":"BERTweet","rank_in_archive_order":3,"of":22,"metrics":{"AUROC":"0.979","GMB BNSP":"0.9603","GMB BPSN":"0.8945","GMB Subgroup":"0.878","Macro F1":"0.3612","Micro F1":"0.4928","Precision":"0.3363","Recall":"0.9216"},"uses_additional_data":false},{"leaderboard":"/sota/toxic-comment-classification-on-civil","task":"Toxic Comment Classification","dataset":"Civil Comments","model":"HateBERT","rank_in_archive_order":4,"of":22,"metrics":{"AUROC":"0.9791","GMB BNSP":"0.9589","GMB BPSN":"0.8915","GMB Subgroup":"0.8744","Macro F1":"0.3679","Micro F1":"0.4844","Precision":"0.3297","Recall":"0.9165"},"uses_additional_data":false},{"leaderboard":"/sota/toxic-comment-classification-on-civil","task":"Toxic Comment Classification","dataset":"Civil Comments","model":"RoBERTa BCE","rank_in_archive_order":5,"of":22,"metrics":{"AUROC":"0.9813","GMB BNSP":"0.9616","GMB BPSN":"0.8901","GMB Subgroup":"0.88","Macro F1":"0.4749","Micro F1":"0.5359","Precision":"0.3836","Recall":"0.8891"},"uses_additional_data":false},{"leaderboard":"/sota/toxic-comment-classification-on-civil","task":"Toxic Comment Classification","dataset":"Civil Comments","model":"XLM RoBERTa","rank_in_archive_order":6,"of":22,"metrics":{"GMB BPSN":"0.8859","Micro F1":"0.468","Precision":"0.3135","Recall":"0.923"},"uses_additional_data":false},{"leaderboard":"/sota/toxic-comment-classification-on-civil","task":"Toxic Comment Classification","dataset":"Civil Comments","model":"XLNet","rank_in_archive_order":7,"of":22,"metrics":{"GMB BNSP":"0.9597","GMB BPSN":"0.8834","GMB Subgroup":"0.8689","Macro F1":"0.3336","Micro F1":"0.4586","Precision":"0.3045","Recall":"0.9254"},"uses_additional_data":false},{"leaderboard":"/sota/toxic-comment-classification-on-civil","task":"Toxic Comment Classification","dataset":"Civil Comments","model":"DistilBERT","rank_in_archive_order":8,"of":22,"metrics":{"AUROC":"0.9804","GMB BNSP":"0.9644","GMB BPSN":"0.874","GMB Subgroup":"0.8762","Macro F1":"0.3879","Micro F1":"0.5115","Precision":"0.3572","Recall":"0.9001"},"uses_additional_data":false},{"leaderboard":"/sota/toxic-comment-classification-on-civil","task":"Toxic Comment Classification","dataset":"Civil Comments","model":"BiGRU","rank_in_archive_order":9,"of":22,"metrics":{"GMB BPSN":"0.8616"},"uses_additional_data":false},{"leaderboard":"/sota/toxic-comment-classification-on-civil","task":"Toxic Comment Classification","dataset":"Civil Comments","model":"Unfreeze Glove ResNet 44","rank_in_archive_order":10,"of":22,"metrics":{"AUROC":"0.966","GMB BPSN":"0.8493","GMB Subgroup":"0.8421","Macro F1":"0.4648","Micro F1":"0.5958","Precision":"0.4835","Recall":"0.7759"},"uses_additional_data":false},{"leaderboard":"/sota/toxic-comment-classification-on-civil","task":"Toxic Comment Classification","dataset":"Civil Comments","model":"Unfreeze Glove ResNet 56","rank_in_archive_order":11,"of":22,"metrics":{"AUROC":"0.9639","GMB BPSN":"0.8445","GMB Subgroup":"0.8487","Macro F1":"0.3778","Recall":"0.8707"},"uses_additional_data":false},{"leaderboard":"/sota/toxic-comment-classification-on-civil","task":"Toxic Comment Classification","dataset":"Civil Comments","model":"Compact Convolutional Transformer (CCT)","rank_in_archive_order":12,"of":22,"metrics":{"AUROC":"0.9526","GMB BNSP":"0.9447","GMB BPSN":"0.8307","GMB Subgroup":"0.8133","Macro F1":"0.3428","Micro F1":"0.4874","Precision":"0.3507","Recall":"0.7983"},"uses_additional_data":false},{"leaderboard":"/sota/toxic-comment-classification-on-civil","task":"Toxic Comment Classification","dataset":"Civil Comments","model":"Freeze Glove ResNet 44","rank_in_archive_order":13,"of":22,"metrics":{"GMB BPSN":"0.7876","GMB Subgroup":"0.8219","Macro F1":"0.4189","Micro F1":"0.5591","Precision":"0.4631","Recall":"0.7053"},"uses_additional_data":false},{"leaderboard":"/sota/toxic-comment-classification-on-civil","task":"Toxic Comment Classification","dataset":"Civil Comments","model":"BiLSTM","rank_in_archive_order":14,"of":22,"metrics":{"GMB Subgroup":"0.8636","Micro F1":"0.5115","Precision":"0.3572"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2301.11125","mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}