{"url":"/dataset/demetr","name":"Demetr","full_name":null,"description_markdown":"**Demetr** is a diagnostic dataset with 31K English examples (translated from 10 source languages) for evaluating the sensitivity of MT evaluation metrics to 35 different linguistic perturbations spanning semantic, syntactic, and morphological error categories.\r\n\r\nSource: [DEMETR: Diagnosing Evaluation Metrics for Translation](https://arxiv.org/pdf/2210.13746v1.pdf)\r\n\r\nImage Source: [https://arxiv.org/pdf/2210.13746v1.pdf](https://arxiv.org/pdf/2210.13746v1.pdf)","description_withheld":null,"homepage":"https://github.com/marzenakrp/demetr","introduced_date":"2022-10-25","introduced_date_note":null,"introduced_by":{"paper":"/paper/demetr-diagnosing-evaluation-metrics-for","title":"DEMETR: Diagnosing Evaluation Metrics for Translation","first_author":"Marzena Karpinska","url":null},"license":{"name":"MIT License","url":"https://github.com/marzenakrp/demetr/blob/main/LICENSE"},"modalities":[{"name":"Texts","url":"/datasets/modality/texts"}],"tasks":[{"name":"Machine Translation","url":"/task/machine-translation","datasets_with_task":"/datasets/task/machine-translation"},{"name":"Translation","url":"/task/translation","datasets_with_task":"/datasets/task/translation"}],"languages":[{"name":"English","url":"/datasets/language/english"},{"name":"French","url":"/datasets/language/french"},{"name":"Spanish","url":"/datasets/language/spanish"},{"name":"German","url":"/datasets/language/german"},{"name":"Italian","url":"/datasets/language/italian"},{"name":"Chinese","url":"/datasets/language/chinese"},{"name":"Japanese","url":"/datasets/language/japanese"},{"name":"Russian","url":"/datasets/language/russian"},{"name":"Czech","url":"/datasets/language/czech"},{"name":"Hindi","url":"/datasets/language/hindi"},{"name":"Polish","url":"/datasets/language/polish"}],"variants":["Demetr"],"data_loaders":[],"num_papers_in_archive":7,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28"},"benchmarks":[],"papers_with_a_benchmark_row":[],"syntology_totals":{"read_at":"2026-09-25T09:33:49+00:00","papers_with_samples":0,"samples_harvested":0,"samples_ran":0,"samples_unverified":0,"pointer_only_for_licence":0,"papers_with_no_sample_that_ran":0,"note":"the per-paper counts above, summed; not a rate"},"papers_note":"The archive never published its papers-using-dataset list; these are papers with a leaderboard row on this dataset's benchmarks."}