{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/metaphorical-user-simulators-for-evaluating","title":"Metaphorical User Simulators for Evaluating Task-oriented Dialogue Systems","arxiv_id":"2204.00763","date":"2022-04-02","proceeding":null,"authors":["Weiwei Sun","Shuyu Guo","Shuo Zhang","Pengjie Ren","Zhumin Chen","Maarten de Rijke","Zhaochun Ren"],"abstract":"Task-oriented dialogue systems (TDSs) are assessed mainly in an offline setting or through human evaluation. The evaluation is often limited to single-turn or is very time-intensive. As an alternative, user simulators that mimic user behavior allow us to consider a broad set of user goals to generate human-like conversations for simulated evaluation. Employing existing user simulators to evaluate TDSs is challenging as user simulators are primarily designed to optimize dialogue policies for TDSs and have limited evaluation capabilities. Moreover, the evaluation of user simulators is an open challenge. In this work, we propose a metaphorical user simulator for end-to-end TDS evaluation, where we define a simulator to be metaphorical if it simulates user's analogical thinking in interactions with systems. We also propose a tester-based evaluation framework to generate variants, i.e., dialogue systems with different capabilities. Our user simulator constructs a metaphorical user model that assists the simulator in reasoning by referring to prior knowledge when encountering new items. We estimate the quality of simulators by checking the simulated interactions between simulators and variants. Our experiments are conducted using three TDS datasets. The proposed user simulator demonstrates better consistency with manual evaluation than an agenda-based simulator and a seq2seq model on three datasets; our tester framework demonstrates efficiency and has been tested on multiple tasks, such as conversational recommendation and e-commerce dialogues.","url_abs":"https://arxiv.org/abs/2204.00763v5","url_pdf":"https://arxiv.org/pdf/2204.00763v5.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"metaphorical-user-simulators-for-evaluating","repo_url":"https://github.com/superbooming/simtester","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"metaphorical-user-simulators-for-evaluating","repo_url":"https://github.com/sunnweiwei/metasim","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"conversational-recommendation","task_name":"Conversational Recommendation"},{"task_slug":"task-oriented-dialogue-systems","task_name":"Task-Oriented Dialogue Systems"}],"methods":[{"method_slug":"lstm","method_name":"LSTM"},{"method_slug":"seq2seq","method_name":"Seq2Seq"},{"method_slug":"sigmoid-activation","method_name":"Sigmoid Activation"},{"method_slug":"tanh-activation","method_name":"Tanh Activation"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2204.00763","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2204.00763"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/superbooming/simtester","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sunnweiwei/metasim","reach":null}],"summary":{"ran_draft_wrong":3,"unverified":6},"by_repo_kind":{"official":{"samples":9,"ran":3,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":2,"samples":[{"code_sha256_prefix":"5cbda0fae871f21c","entry":"parse_act","repo":"sunnweiwei/metasim","repo_kind":"official","path":"metasim/train_mwoz.py","file_url":"https://github.com/sunnweiwei/metasim/blob/HEAD/metasim/train_mwoz.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5cbda0fae871f21c"}},{"code_sha256_prefix":"a4875d6233744ce4","entry":"parse_goal","repo":"sunnweiwei/metasim","repo_kind":"official","path":"metasim/train_mwoz.py","file_url":"https://github.com/sunnweiwei/metasim/blob/HEAD/metasim/train_mwoz.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a4875d6233744ce4"}},{"code_sha256_prefix":"af64ec220e8bcdbc","entry":"url_to_filename","repo":"superbooming/simtester","repo_kind":"official","path":"simtester/utils/allennlp_file_utils.py","file_url":"https://github.com/superbooming/simtester/blob/HEAD/simtester/utils/allennlp_file_utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"af64ec220e8bcdbc"}},{"code_sha256_prefix":"33d45ac6e8fe7c98","entry":"cached_path","repo":"superbooming/simtester","repo_kind":"official","path":"simtester/utils/allennlp_file_utils.py","file_url":"https://github.com/superbooming/simtester/blob/HEAD/simtester/utils/allennlp_file_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"33d45ac6e8fe7c98"}},{"code_sha256_prefix":"4b48138f6d5166a1","entry":"clean_time","repo":"superbooming/simtester","repo_kind":"official","path":"simtester/agent/damd/clean_dataset.py","file_url":"https://github.com/superbooming/simtester/blob/HEAD/simtester/agent/damd/clean_dataset.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4b48138f6d5166a1"}},{"code_sha256_prefix":"8ecbc06b66b54bcb","entry":"filename_to_url","repo":"superbooming/simtester","repo_kind":"official","path":"simtester/utils/allennlp_file_utils.py","file_url":"https://github.com/superbooming/simtester/blob/HEAD/simtester/utils/allennlp_file_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8ecbc06b66b54bcb"}},{"code_sha256_prefix":"7f8e39857c0f1353","entry":"load_yaml_configs","repo":"superbooming/simtester","repo_kind":"official","path":"simtester/utils/utils.py","file_url":"https://github.com/superbooming/simtester/blob/HEAD/simtester/utils/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7f8e39857c0f1353"}},{"code_sha256_prefix":"b3d876f28f4ed5bd","entry":"mp","repo":"superbooming/simtester","repo_kind":"official","path":"simtester/utils/utils.py","file_url":"https://github.com/superbooming/simtester/blob/HEAD/simtester/utils/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b3d876f28f4ed5bd"}},{"code_sha256_prefix":"d2c7e0088c58b4ba","entry":"run_shell","repo":"superbooming/simtester","repo_kind":"official","path":"simtester/utils/utils.py","file_url":"https://github.com/superbooming/simtester/blob/HEAD/simtester/utils/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d2c7e0088c58b4ba"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}