{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/fabricator-an-open-source-toolkit-for","title":"Fabricator: An Open Source Toolkit for Generating Labeled Training Data with Teacher LLMs","arxiv_id":"2309.09582","date":"2023-09-18","proceeding":null,"authors":["Jonas Golde","Patrick Haller","Felix Hamborg","Julian Risch","Alan Akbik"],"abstract":"Most NLP tasks are modeled as supervised learning and thus require labeled training data to train effective models. However, manually producing such data at sufficient quality and quantity is known to be costly and time-intensive. Current research addresses this bottleneck by exploring a novel paradigm called zero-shot learning via dataset generation. Here, a powerful LLM is prompted with a task description to generate labeled data that can be used to train a downstream NLP model. For instance, an LLM might be prompted to \"generate 500 movie reviews with positive overall sentiment, and another 500 with negative sentiment.\" The generated data could then be used to train a binary sentiment classifier, effectively leveraging an LLM as a teacher to a smaller student model. With this demo, we introduce Fabricator, an open-source Python toolkit for dataset generation. Fabricator implements common dataset generation workflows, supports a wide range of downstream NLP tasks (such as text classification, question answering, and entity recognition), and is integrated with well-known libraries to facilitate quick experimentation. With Fabricator, we aim to support researchers in conducting reproducible dataset generation experiments using LLMs and help practitioners apply this approach to train models for downstream tasks.","url_abs":"https://arxiv.org/abs/2309.09582v2","url_pdf":"https://arxiv.org/pdf/2309.09582v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"fabricator-an-open-source-toolkit-for","repo_url":"https://github.com/flairnlp/fabricator","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"dataset-generation","task_name":"Dataset Generation"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"text-classification","task_name":"Text Classification"},{"task_slug":"zero-shot-learning","task_name":"Zero-Shot Learning"},{"task_slug":"text-classification-1","task_name":"text-classification"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2309.09582","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2309.09582"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/flairNLP/fabricator","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/deepset-ai/Haystack","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":17,"unverified":2},"by_repo_kind":{"official":{"samples":13,"ran":11,"repositories":1},"found_in_text":{"samples":6,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"81ab569cd8346156","entry":"calculate_answer_start","repo":"flairNLP/fabricator","repo_kind":"official","path":"src/fabricator/dataset_transformations/question_answering.py","file_url":"https://github.com/flairNLP/fabricator/blob/HEAD/src/fabricator/dataset_transformations/question_answering.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"81ab569cd8346156"}},{"code_sha256_prefix":"a8da3033f9af176d","entry":"convert_spans_to_token_labels","repo":"flairNLP/fabricator","repo_kind":"official","path":"src/fabricator/dataset_transformations/token_classification.py","file_url":"https://github.com/flairNLP/fabricator/blob/HEAD/src/fabricator/dataset_transformations/token_classification.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a8da3033f9af176d"}},{"code_sha256_prefix":"c1f2f10c10a88040","entry":"create_timestamp_path","repo":"flairNLP/fabricator","repo_kind":"official","path":"src/fabricator/utils.py","file_url":"https://github.com/flairNLP/fabricator/blob/HEAD/src/fabricator/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c1f2f10c10a88040"}},{"code_sha256_prefix":"164df86632d7c91b","entry":"generate_qualified_class_name","repo":"deepset-ai/Haystack","repo_kind":"found_in_text","path":"haystack/core/serialization.py","file_url":"https://github.com/deepset-ai/Haystack/blob/HEAD/haystack/core/serialization.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"164df86632d7c91b"}},{"code_sha256_prefix":"cddc89c4c8339cd7","entry":"get_labels_from_dataset","repo":"flairNLP/fabricator","repo_kind":"official","path":"src/fabricator/dataset_transformations/text_classification.py","file_url":"https://github.com/flairNLP/fabricator/blob/HEAD/src/fabricator/dataset_transformations/text_classification.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cddc89c4c8339cd7"}},{"code_sha256_prefix":"6fefe2cdced3efd0","entry":"has_any_trigger","repo":"deepset-ai/Haystack","repo_kind":"found_in_text","path":"haystack/core/pipeline/component_checks.py","file_url":"https://github.com/deepset-ai/Haystack/blob/HEAD/haystack/core/pipeline/component_checks.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6fefe2cdced3efd0"}},{"code_sha256_prefix":"f000dffd4a55b693","entry":"mark_deserialization_internal","repo":"deepset-ai/Haystack","repo_kind":"found_in_text","path":"haystack/core/serialization_security.py","file_url":"https://github.com/deepset-ai/Haystack/blob/HEAD/haystack/core/serialization_security.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f000dffd4a55b693"}},{"code_sha256_prefix":"61479ed8bec405d6","entry":"patch_log_method_to_kwargs_only","repo":"deepset-ai/Haystack","repo_kind":"found_in_text","path":"haystack/logging.py","file_url":"https://github.com/deepset-ai/Haystack/blob/HEAD/haystack/logging.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"61479ed8bec405d6"}},{"code_sha256_prefix":"832a10875cdb79ca","entry":"patch_log_with_level_method_to_kwargs_only","repo":"deepset-ai/Haystack","repo_kind":"found_in_text","path":"haystack/logging.py","file_url":"https://github.com/deepset-ai/Haystack/blob/HEAD/haystack/logging.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"832a10875cdb79ca"}},{"code_sha256_prefix":"c8cb9b84c1356fdd","entry":"patch_make_records_to_use_kwarg_string_interpolation","repo":"deepset-ai/Haystack","repo_kind":"found_in_text","path":"haystack/logging.py","file_url":"https://github.com/deepset-ai/Haystack/blob/HEAD/haystack/logging.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c8cb9b84c1356fdd"}},{"code_sha256_prefix":"85f9d2e4993481da","entry":"postprocess_squad_format","repo":"flairNLP/fabricator","repo_kind":"official","path":"src/fabricator/dataset_transformations/question_answering.py","file_url":"https://github.com/flairNLP/fabricator/blob/HEAD/src/fabricator/dataset_transformations/question_answering.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"85f9d2e4993481da"}},{"code_sha256_prefix":"e322caa231cc0084","entry":"preprocess_squad_format","repo":"flairNLP/fabricator","repo_kind":"official","path":"src/fabricator/dataset_transformations/question_answering.py","file_url":"https://github.com/flairNLP/fabricator/blob/HEAD/src/fabricator/dataset_transformations/question_answering.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e322caa231cc0084"}},{"code_sha256_prefix":"1266726c77cca097","entry":"random_sampler","repo":"flairNLP/fabricator","repo_kind":"official","path":"src/fabricator/samplers/samplers.py","file_url":"https://github.com/flairNLP/fabricator/blob/HEAD/src/fabricator/samplers/samplers.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1266726c77cca097"}},{"code_sha256_prefix":"7fba20d4a7dfc7a3","entry":"replace_class_labels","repo":"flairNLP/fabricator","repo_kind":"official","path":"src/fabricator/dataset_transformations/text_classification.py","file_url":"https://github.com/flairNLP/fabricator/blob/HEAD/src/fabricator/dataset_transformations/text_classification.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7fba20d4a7dfc7a3"}},{"code_sha256_prefix":"d5e8675475cdb42a","entry":"replace_token_labels","repo":"flairNLP/fabricator","repo_kind":"official","path":"src/fabricator/dataset_transformations/token_classification.py","file_url":"https://github.com/flairNLP/fabricator/blob/HEAD/src/fabricator/dataset_transformations/token_classification.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d5e8675475cdb42a"}},{"code_sha256_prefix":"bcd2a038c58aa493","entry":"single_label_stratified_sample","repo":"flairNLP/fabricator","repo_kind":"official","path":"src/fabricator/samplers/samplers.py","file_url":"https://github.com/flairNLP/fabricator/blob/HEAD/src/fabricator/samplers/samplers.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bcd2a038c58aa493"}},{"code_sha256_prefix":"e560cf2ebbd575b2","entry":"single_label_task_sampler","repo":"flairNLP/fabricator","repo_kind":"official","path":"src/fabricator/samplers/samplers.py","file_url":"https://github.com/flairNLP/fabricator/blob/HEAD/src/fabricator/samplers/samplers.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e560cf2ebbd575b2"}},{"code_sha256_prefix":"a5bb802ee94c5178","entry":"convert_label_ids_to_texts","repo":"flairNLP/fabricator","repo_kind":"official","path":"src/fabricator/dataset_transformations/text_classification.py","file_url":"https://github.com/flairNLP/fabricator/blob/HEAD/src/fabricator/dataset_transformations/text_classification.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a5bb802ee94c5178"}},{"code_sha256_prefix":"b08e9d3cfc7a721d","entry":"convert_token_labels_to_spans","repo":"flairNLP/fabricator","repo_kind":"official","path":"src/fabricator/dataset_transformations/token_classification.py","file_url":"https://github.com/flairNLP/fabricator/blob/HEAD/src/fabricator/dataset_transformations/token_classification.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b08e9d3cfc7a721d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}