{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/wic-tsv-an-evaluation-benchmark-for-target","title":"WiC-TSV: An Evaluation Benchmark for Target Sense Verification of Words in Context","arxiv_id":"2004.15016","date":"2020-04-30","proceeding":"EACL 2021 2","authors":["Anna Breit","Artem Revenko","Kiamehr Rezaee","Mohammad Taher Pilehvar","Jose Camacho-Collados"],"abstract":"We present WiC-TSV, a new multi-domain evaluation benchmark for Word Sense Disambiguation. More specifically, we introduce a framework for Target Sense Verification of Words in Context which grounds its uniqueness in the formulation as a binary classification task thus being independent of external sense inventories, and the coverage of various domains. This makes the dataset highly flexible for the evaluation of a diverse set of models and systems in and across domains. WiC-TSV provides three different evaluation settings, depending on the input signals provided to the model. We set baseline performance on the dataset using state-of-the-art language models. Experimental results show that even though these models can perform decently on the task, there remains a gap between machine and human performance, especially in out-of-domain settings. WiC-TSV data is available at https://competitions.codalab.org/competitions/23683","url_abs":"https://arxiv.org/abs/2004.15016v3","url_pdf":"https://arxiv.org/pdf/2004.15016v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"wic-tsv-an-evaluation-benchmark-for-target","repo_url":"https://github.com/semantic-web-company/wic-tsv","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"binary-classification","task_name":"Binary Classification"},{"task_slug":"entity-linking","task_name":"Entity Linking"},{"task_slug":"word-sense-disambiguation","task_name":"Word Sense Disambiguation"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[{"slug":"wic-tsv","name":"WiC-TSV","full_name":"Words-in-Context: Target Sense Verification"}],"methods_introduced":[],"results":[{"leaderboard":"/sota/entity-linking-on-wic-tsv","task":"Entity Linking","dataset":"WiC-TSV","model":"Bert-base","rank_in_archive_order":4,"of":8,"metrics":{"Task 1 Accuracy: all":"75.3","Task 1 Accuracy: domain specific":"77.9","Task 1 Accuracy: general purpose":"73.3","Task 2 Accuracy: all":"71.7","Task 2 Accuracy: domain specific":"74.7","Task 2 Accuracy: general purpose":"68.6","Task 3 Accuracy: all":"76.6","Task 3 Accuracy: domain specific":"80.4","Task 3 Accuracy: general purpose":"73.5"},"uses_additional_data":false},{"leaderboard":"/sota/entity-linking-on-wic-tsv","task":"Entity Linking","dataset":"WiC-TSV","model":"Unsupervised Bert","rank_in_archive_order":5,"of":8,"metrics":{"Task 1 Accuracy: all":"54.4","Task 1 Accuracy: domain specific":"60.6","Task 1 Accuracy: general purpose":"49.2","Task 2 Accuracy: all":"62.8","Task 2 Accuracy: domain specific":"69.1","Task 2 Accuracy: general purpose":"57.6","Task 3 Accuracy: all":"60.5","Task 3 Accuracy: domain specific":"67.9","Task 3 Accuracy: general purpose":"54.4"},"uses_additional_data":false},{"leaderboard":"/sota/entity-linking-on-wic-tsv","task":"Entity Linking","dataset":"WiC-TSV","model":"FastText","rank_in_archive_order":6,"of":8,"metrics":{"Task 1 Accuracy: all":"53.7","Task 1 Accuracy: domain specific":"50.6","Task 1 Accuracy: general purpose":"56.2","Task 2 Accuracy: all":"52.7","Task 2 Accuracy: domain specific":"47.7","Task 2 Accuracy: general purpose":"56.8","Task 3 Accuracy: all":"53.4","Task 3 Accuracy: domain specific":"49.0","Task 3 Accuracy: general purpose":"57.1"},"uses_additional_data":false},{"leaderboard":"/sota/entity-linking-on-wic-tsv","task":"Entity Linking","dataset":"WiC-TSV","model":"All true","rank_in_archive_order":7,"of":8,"metrics":{"Task 1 Accuracy: all":"50.8","Task 1 Accuracy: domain specific":"47.0","Task 1 Accuracy: general purpose":"53.8","Task 2 Accuracy: all":"50.8","Task 2 Accuracy: domain specific":"47.0","Task 2 Accuracy: general purpose":"53.8","Task 3 Accuracy: all":"50.8","Task 3 Accuracy: domain specific":"47.0","Task 3 Accuracy: general purpose":"53.8"},"uses_additional_data":false},{"leaderboard":"/sota/entity-linking-on-wic-tsv","task":"Entity Linking","dataset":"WiC-TSV","model":"Human","rank_in_archive_order":8,"of":8,"metrics":{"Task 3 Accuracy: all":"85.3","Task 3 Accuracy: domain specific":"89.2","Task 3 Accuracy: general purpose":"82.1"},"uses_additional_data":false},{"leaderboard":"/sota/word-sense-disambiguation-on-wic-tsv","task":"Word Sense Disambiguation","dataset":"WiC-TSV","model":"Bert-base","rank_in_archive_order":4,"of":8,"metrics":{"Task 1 Accuracy: all":"75.3","Task 1 Accuracy: domain specific":"77.9","Task 1 Accuracy: general purpose":"73.3","Task 2 Accuracy: all":"71.7","Task 2 Accuracy: domain specific":"74.7","Task 2 Accuracy: general purpose":"68.6","Task 3 Accuracy: all":"76.6","Task 3 Accuracy: domain specific":"80.4","Task 3 Accuracy: general purpose":"73.5"},"uses_additional_data":false},{"leaderboard":"/sota/word-sense-disambiguation-on-wic-tsv","task":"Word Sense Disambiguation","dataset":"WiC-TSV","model":"Unsupervised Bert","rank_in_archive_order":5,"of":8,"metrics":{"Task 1 Accuracy: all":"54.4","Task 1 Accuracy: domain specific":"60.6","Task 1 Accuracy: general purpose":"49.2","Task 2 Accuracy: all":"62.8","Task 2 Accuracy: domain specific":"69.1","Task 2 Accuracy: general purpose":"57.6","Task 3 Accuracy: all":"60.5","Task 3 Accuracy: domain specific":"67.9","Task 3 Accuracy: general purpose":"54.4"},"uses_additional_data":false},{"leaderboard":"/sota/word-sense-disambiguation-on-wic-tsv","task":"Word Sense Disambiguation","dataset":"WiC-TSV","model":"FastText","rank_in_archive_order":6,"of":8,"metrics":{"Task 1 Accuracy: all":"53.7","Task 1 Accuracy: domain specific":"50.6","Task 1 Accuracy: general purpose":"56.2","Task 2 Accuracy: all":"52.7","Task 2 Accuracy: domain specific":"47.7","Task 2 Accuracy: general purpose":"56.8","Task 3 Accuracy: all":"53.4","Task 3 Accuracy: domain specific":"49.0","Task 3 Accuracy: general purpose":"57.1"},"uses_additional_data":false},{"leaderboard":"/sota/word-sense-disambiguation-on-wic-tsv","task":"Word Sense Disambiguation","dataset":"WiC-TSV","model":"All true","rank_in_archive_order":7,"of":8,"metrics":{"Task 1 Accuracy: all":"50.8","Task 1 Accuracy: domain specific":"47.0","Task 1 Accuracy: general purpose":"53.8","Task 2 Accuracy: all":"50.8","Task 2 Accuracy: domain specific":"47.0","Task 2 Accuracy: general purpose":"53.8","Task 3 Accuracy: all":"50.8","Task 3 Accuracy: domain specific":"47.0","Task 3 Accuracy: general purpose":"53.8"},"uses_additional_data":false},{"leaderboard":"/sota/word-sense-disambiguation-on-wic-tsv","task":"Word Sense Disambiguation","dataset":"WiC-TSV","model":"Human","rank_in_archive_order":8,"of":8,"metrics":{"Task 3 Accuracy: all":"85.3","Task 3 Accuracy: domain specific":"89.2","Task 3 Accuracy: general purpose":"82.1"},"uses_additional_data":false}],"syntology":{"atlas_url":null,"mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}