{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2509-14030","title":"CrowdAgent: Multi-Agent Managed Multi-Source Annotation System","arxiv_id":"2509.14030","date":"2025-09-17","proceeding":"EMNLP","authors":["Maosheng Qin","Renyu Zhu","Mingxuan Xia","Chenkai Chen","Zhen Zhu","Minmin Lin","Junbo Zhao","Lu Xu","Changjie Fan","Runze Wu","Haobo Wang"],"abstract":"High-quality annotated data is a cornerstone of modern Natural Language Processing (NLP). While recent methods begin to leverage diverse annotation sources-including Large Language Models (LLMs), Small Language Models (SLMs), and human experts-they often focus narrowly on the labeling step itself. A critical gap remains in the holistic process control required to manage these sources dynamically, addressing complex scheduling and quality-cost trade-offs in a unified manner. Inspired by real-world crowdsourcing companies, we introduce CrowdAgent, a multi-agent system that provides end-to-end process control by integrating task assignment, data annotation, and quality/cost management. It implements a novel methodology that rationally assigns tasks, enabling LLMs, SLMs, and human experts to advance synergistically in a collaborative annotation workflow. We demonstrate the effectiveness of CrowdAgent through extensive experiments on six diverse multimodal classification tasks. The source code and video demo are available at https://github.com/QMMMS/CrowdAgent.","url_abs":"https://arxiv.org/abs/2509.14030","url_pdf":"https://arxiv.org/pdf/2509.14030","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2509.14030","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2509.14030"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/QMMMS/CrowdAgent","reach":{"status":"ok"}}],"summary":{"unverified":10},"by_repo_kind":{"found_in_text":{"samples":10,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":10,"samples":[{"code_sha256_prefix":"5730bf7db9196996","entry":"coreset_active_learning","repo":"QMMMS/CrowdAgent","repo_kind":"found_in_text","path":"crowd-agent-backend/src/human_function/human_annotation.py","file_url":"https://github.com/QMMMS/CrowdAgent/blob/HEAD/crowd-agent-backend/src/human_function/human_annotation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5730bf7db9196996"}},{"code_sha256_prefix":"2761c61217ad51cb","entry":"evaluate_model","repo":"QMMMS/CrowdAgent","repo_kind":"found_in_text","path":"crowd-agent-backend/src/slm_function/robust_convnextv2.py","file_url":"https://github.com/QMMMS/CrowdAgent/blob/HEAD/crowd-agent-backend/src/slm_function/robust_convnextv2.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2761c61217ad51cb"}},{"code_sha256_prefix":"71c121cbd87f1278","entry":"get_args","repo":"QMMMS/CrowdAgent","repo_kind":"found_in_text","path":"crowd-agent-backend/src/slm_function/chinese_roberta.py","file_url":"https://github.com/QMMMS/CrowdAgent/blob/HEAD/crowd-agent-backend/src/slm_function/chinese_roberta.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"71c121cbd87f1278"}},{"code_sha256_prefix":"1723edb6f852d86b","entry":"get_error_matrix_dict","repo":"QMMMS/CrowdAgent","repo_kind":"found_in_text","path":"crowd-agent-backend/src/data_function/validation.py","file_url":"https://github.com/QMMMS/CrowdAgent/blob/HEAD/crowd-agent-backend/src/data_function/validation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1723edb6f852d86b"}},{"code_sha256_prefix":"a85bee86ce4bdf8f","entry":"get_task_status","repo":"QMMMS/CrowdAgent","repo_kind":"found_in_text","path":"crowd-agent-frontend/util/crowdagent_tool.py","file_url":"https://github.com/QMMMS/CrowdAgent/blob/HEAD/crowd-agent-frontend/util/crowdagent_tool.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a85bee86ce4bdf8f"}},{"code_sha256_prefix":"8b87422b967cb28f","entry":"ground_truth_annotation","repo":"QMMMS/CrowdAgent","repo_kind":"found_in_text","path":"crowd-agent-backend/src/human_function/human_annotation.py","file_url":"https://github.com/QMMMS/CrowdAgent/blob/HEAD/crowd-agent-backend/src/human_function/human_annotation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"8b87422b967cb28f"}},{"code_sha256_prefix":"dc0e14ace1e12cb9","entry":"read_confusion_matrix","repo":"QMMMS/CrowdAgent","repo_kind":"found_in_text","path":"crowd-agent-backend/src/data_function/validation.py","file_url":"https://github.com/QMMMS/CrowdAgent/blob/HEAD/crowd-agent-backend/src/data_function/validation.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"dc0e14ace1e12cb9"}},{"code_sha256_prefix":"ae661e1efdd0c818","entry":"set_logger","repo":"QMMMS/CrowdAgent","repo_kind":"found_in_text","path":"crowd-agent-backend/src/llm_function/active_labeling_seq.py","file_url":"https://github.com/QMMMS/CrowdAgent/blob/HEAD/crowd-agent-backend/src/llm_function/active_labeling_seq.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ae661e1efdd0c818"}},{"code_sha256_prefix":"76c0428ef9797ab2","entry":"set_logger","repo":"QMMMS/CrowdAgent","repo_kind":"found_in_text","path":"crowd-agent-backend/src/slm_function/chinese_roberta.py","file_url":"https://github.com/QMMMS/CrowdAgent/blob/HEAD/crowd-agent-backend/src/slm_function/chinese_roberta.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"76c0428ef9797ab2"}},{"code_sha256_prefix":"9f5cb063df74b4d7","entry":"validate_on_labeled_gt","repo":"QMMMS/CrowdAgent","repo_kind":"found_in_text","path":"crowd-agent-backend/src/slm_function/chinese_roberta.py","file_url":"https://github.com/QMMMS/CrowdAgent/blob/HEAD/crowd-agent-backend/src/slm_function/chinese_roberta.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9f5cb063df74b4d7"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.AI","source":"arxiv_api"},"syntology_extracted_results":null}