{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/how-not-to-evaluate-your-dialogue-system-an","title":"How NOT To Evaluate Your Dialogue System: An Empirical Study of Unsupervised Evaluation Metrics for Dialogue Response Generation","arxiv_id":"1603.08023","date":"2016-03-25","proceeding":"EMNLP 2016 11","authors":["Chia-Wei Liu","Ryan Lowe","Iulian V. Serban","Michael Noseworthy","Laurent Charlin","Joelle Pineau"],"abstract":"We investigate evaluation metrics for dialogue response generation systems\nwhere supervised labels, such as task completion, are not available. Recent\nworks in response generation have adopted metrics from machine translation to\ncompare a model's generated response to a single target response. We show that\nthese metrics correlate very weakly with human judgements in the non-technical\nTwitter domain, and not at all in the technical Ubuntu domain. We provide\nquantitative and qualitative results highlighting specific weaknesses in\nexisting metrics, and provide recommendations for future development of better\nautomatic evaluation metrics for dialogue systems.","url_abs":"http://arxiv.org/abs/1603.08023v2","url_pdf":"http://arxiv.org/pdf/1603.08023v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"how-not-to-evaluate-your-dialogue-system-an","repo_url":"https://github.com/23LuZ/the-Evaluation-of-ChitChat-System","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"how-not-to-evaluate-your-dialogue-system-an","repo_url":"https://github.com/piekey1994/IOM","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"machine-translation","task_name":"Machine Translation"},{"task_slug":"response-generation","task_name":"Response Generation"},{"task_slug":"translation","task_name":"Translation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1603.08023","mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}