{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/a-surprisingly-robust-trick-for-winograd","title":"A Surprisingly Robust Trick for Winograd Schema Challenge","arxiv_id":"1905.06290","date":"2019-05-15","proceeding":null,"authors":["Vid Kocijan","Ana-Maria Cretu","Oana-Maria Camburu","Yordan Yordanov","Thomas Lukasiewicz"],"abstract":"The Winograd Schema Challenge (WSC) dataset WSC273 and its inference counterpart WNLI are popular benchmarks for natural language understanding and commonsense reasoning. In this paper, we show that the performance of three language models on WSC273 strongly improves when fine-tuned on a similar pronoun disambiguation problem dataset (denoted WSCR). We additionally generate a large unsupervised WSC-like dataset. By fine-tuning the BERT language model both on the introduced and on the WSCR dataset, we achieve overall accuracies of 72.5% and 74.7% on WSC273 and WNLI, improving the previous state-of-the-art solutions by 8.8% and 9.6%, respectively. Furthermore, our fine-tuned models are also consistently more robust on the \"complex\" subsets of WSC273, introduced by Trichelair et al. (2018).","url_abs":"https://arxiv.org/abs/1905.06290v2","url_pdf":"https://arxiv.org/pdf/1905.06290v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"a-surprisingly-robust-trick-for-winograd","repo_url":"https://github.com/vid-koci/bert-commonsense","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"a-surprisingly-robust-trick-for-winograd","repo_url":"https://github.com/TangJiaLong/Knowledge-Projection-for-ERE","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"common-sense-reasoning","task_name":"Common Sense Reasoning"},{"task_slug":"coreference-resolution","task_name":"Coreference Resolution"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"natural-language-inference","task_name":"Natural Language Inference"},{"task_slug":"natural-language-understanding","task_name":"Natural Language Understanding"},{"task_slug":"wnli","task_name":"WNLI"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/coreference-resolution-on-winograd-schema","task":"Coreference Resolution","dataset":"Winograd Schema Challenge","model":"BERTwiki 340M (fine-tuned on WSCR)","rank_in_archive_order":30,"of":82,"metrics":{"Accuracy":"72.5"},"uses_additional_data":false},{"leaderboard":"/sota/coreference-resolution-on-winograd-schema","task":"Coreference Resolution","dataset":"Winograd Schema Challenge","model":"BERT-large 340M (fine-tuned on WSCR)","rank_in_archive_order":32,"of":82,"metrics":{"Accuracy":"71.4"},"uses_additional_data":false},{"leaderboard":"/sota/coreference-resolution-on-winograd-schema","task":"Coreference Resolution","dataset":"Winograd Schema Challenge","model":"BERTwiki 340M (fine-tuned on half of WSCR)","rank_in_archive_order":34,"of":82,"metrics":{"Accuracy":"70.3"},"uses_additional_data":false},{"leaderboard":"/sota/coreference-resolution-on-winograd-schema","task":"Coreference Resolution","dataset":"Winograd Schema Challenge","model":"BERT-base 110M (fine-tuned on WSCR)","rank_in_archive_order":51,"of":82,"metrics":{"Accuracy":"62.3"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-wnli","task":"Natural Language Inference","dataset":"WNLI","model":"BERTwiki 340M (fine-tuned on WSCR)","rank_in_archive_order":13,"of":23,"metrics":{"Accuracy":"74.7"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-wnli","task":"Natural Language Inference","dataset":"WNLI","model":"BERT-large 340M (fine-tuned on WSCR)","rank_in_archive_order":15,"of":23,"metrics":{"Accuracy":"71.9"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-wnli","task":"Natural Language Inference","dataset":"WNLI","model":"BERT-base 110M (fine-tuned on WSCR)","rank_in_archive_order":16,"of":23,"metrics":{"Accuracy":"70.5"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1905.06290","mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}