{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/crwiz-a-framework-for-crowdsourcing-real-time","title":"CRWIZ: A Framework for Crowdsourcing Real-Time Wizard-of-Oz Dialogues","arxiv_id":"2003.05995","date":"2020-03-12","proceeding":"LREC 2020 5","authors":["Francisco J. Chiyah Garcia","José Lopes","Xingkun Liu","Helen Hastie"],"abstract":"Large corpora of task-based and open-domain conversational dialogues are hugely valuable in the field of data-driven dialogue systems. Crowdsourcing platforms, such as Amazon Mechanical Turk, have been an effective method for collecting such large amounts of data. However, difficulties arise when task-based dialogues require expert domain knowledge or rapid access to domain-relevant information, such as databases for tourism. This will become even more prevalent as dialogue systems become increasingly ambitious, expanding into tasks with high levels of complexity that require collaboration and forward planning, such as in our domain of emergency response. In this paper, we propose CRWIZ: a framework for collecting real-time Wizard of Oz dialogues through crowdsourcing for collaborative, complex tasks. This framework uses semi-guided dialogue to avoid interactions that breach procedures and processes only known to experts, while enabling the capture of a wide variety of interactions. The framework is available at https://github.com/JChiyah/crwiz","url_abs":"https://arxiv.org/abs/2003.05995v1","url_pdf":"https://arxiv.org/pdf/2003.05995v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"crwiz-a-framework-for-crowdsourcing-real-time","repo_url":"https://github.com/JChiyah/crwiz","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"none","reach":{"status":"ok","spdx":"BSD-3-Clause"}}],"tasks":[],"methods":[{"method_slug":"wizard","method_name":"Wizard"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2003.05995","atlas_url":"https://app.syntology.ai/?focus=2003.05995","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2003.05995"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/JChiyah/crwiz","reach":{"status":"ok","spdx":"BSD-3-Clause"}}],"summary":{"ran":2,"unverified":5},"by_repo_kind":{"official":{"samples":7,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"4ab72a7fbdc3e9a3","entry":"match_utterance_to_state_formulation","repo":"JChiyah/crwiz","repo_kind":"official","path":"app/crwiz/utils/helper.py","file_url":"https://github.com/JChiyah/crwiz/blob/HEAD/app/crwiz/utils/helper.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"4ab72a7fbdc3e9a3"}},{"code_sha256_prefix":"303694cee996f344","entry":"state_contains_action","repo":"JChiyah/crwiz","repo_kind":"official","path":"app/crwiz/fake_actions.py","file_url":"https://github.com/JChiyah/crwiz/blob/HEAD/app/crwiz/fake_actions.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"303694cee996f344"}},{"code_sha256_prefix":"7421270ead42ade4","entry":"filter_used_transitions","repo":"JChiyah/crwiz","repo_kind":"official","path":"app/crwiz/finite_state_machine.py","file_url":"https://github.com/JChiyah/crwiz/blob/HEAD/app/crwiz/finite_state_machine.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"7421270ead42ade4"}},{"code_sha256_prefix":"d7ce4615a8831b14","entry":"get_room_logs","repo":"JChiyah/crwiz","repo_kind":"official","path":"app/crwiz/utils/log_utils.py","file_url":"https://github.com/JChiyah/crwiz/blob/HEAD/app/crwiz/utils/log_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"d7ce4615a8831b14"}},{"code_sha256_prefix":"be30b361a9f699b0","entry":"get_user_logs","repo":"JChiyah/crwiz","repo_kind":"official","path":"app/crwiz/utils/log_utils.py","file_url":"https://github.com/JChiyah/crwiz/blob/HEAD/app/crwiz/utils/log_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"be30b361a9f699b0"}},{"code_sha256_prefix":"741a2ed3a398fe66","entry":"load_dialogue_states","repo":"JChiyah/crwiz","repo_kind":"official","path":"app/crwiz/dialogue_state.py","file_url":"https://github.com/JChiyah/crwiz/blob/HEAD/app/crwiz/dialogue_state.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"741a2ed3a398fe66"}},{"code_sha256_prefix":"a7db20b99d43d14b","entry":"perform_post_task_analysis","repo":"JChiyah/crwiz","repo_kind":"official","path":"app/crwiz/utils/post_task_analysis.py","file_url":"https://github.com/JChiyah/crwiz/blob/HEAD/app/crwiz/utils/post_task_analysis.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"a7db20b99d43d14b"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}