{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/a-little-human-data-goes-a-long-way","title":"A Little Human Data Goes A Long Way","arxiv_id":"2410.13098","date":"2024-10-17","proceeding":null,"authors":["Dhananjay Ashok","Jonathan May"],"abstract":"Faced with an expensive human annotation process, creators of NLP systems increasingly turn to synthetic data generation. While this method shows promise, the extent to which synthetic data can replace human annotation is poorly understood. We investigate the use of synthetic data in Fact Verification (FV) and Question Answering (QA) by studying the effects of incrementally replacing human generated data with synthetic points on eight diverse datasets. Strikingly, replacing up to 90% of the training data only marginally decreases performance, but replacing the final 10% leads to severe declines. We find that models trained on purely synthetic data can be reliably improved by including as few as 125 human generated data points. We show that matching the performance gain of just a little additional human data (only 200 points) requires an order of magnitude more synthetic data and estimate price ratios at which human annotation would be a more cost-effective solution. Our results suggest that even when human annotation at scale is infeasible, there is great value to having a small proportion of the dataset being human generated.","url_abs":"https://arxiv.org/abs/2410.13098v1","url_pdf":"https://arxiv.org/pdf/2410.13098v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"a-little-human-data-goes-a-long-way","repo_url":"https://github.com/dhananjayashok/littlehumandata","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"fact-verification","task_name":"Fact Verification"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"synthetic-data-generation","task_name":"Synthetic Data Generation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2410.13098","atlas_url":"https://app.syntology.ai/?focus=2410.13098","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.13098"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/dhananjayashok/littlehumandata","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":15,"unverified":2},"by_repo_kind":{"official":{"samples":17,"ran":15,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"100237b0db550ff3","entry":"compute_inclusion","repo":"dhananjayashok/littlehumandata","repo_kind":"official","path":"compute_qa_metrics.py","file_url":"https://github.com/dhananjayashok/littlehumandata/blob/HEAD/compute_qa_metrics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"100237b0db550ff3"}},{"code_sha256_prefix":"dca52e1f7f4a7dbc","entry":"compute_mode_ngrams","repo":"dhananjayashok/littlehumandata","repo_kind":"official","path":"analysis.py","file_url":"https://github.com/dhananjayashok/littlehumandata/blob/HEAD/analysis.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dca52e1f7f4a7dbc"}},{"code_sha256_prefix":"29bacdfb84b03880","entry":"create_conversation","repo":"dhananjayashok/littlehumandata","repo_kind":"official","path":"llm-utils/data_script.py","file_url":"https://github.com/dhananjayashok/littlehumandata/blob/HEAD/llm-utils/data_script.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"29bacdfb84b03880"}},{"code_sha256_prefix":"5c01635c26b2b6f5","entry":"do_evaluation","repo":"dhananjayashok/littlehumandata","repo_kind":"official","path":"llm-utils/common_utils.py","file_url":"https://github.com/dhananjayashok/littlehumandata/blob/HEAD/llm-utils/common_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5c01635c26b2b6f5"}},{"code_sha256_prefix":"c4bde9e27f99624d","entry":"get_df","repo":"dhananjayashok/littlehumandata","repo_kind":"official","path":"llm-utils/predict.py","file_url":"https://github.com/dhananjayashok/littlehumandata/blob/HEAD/llm-utils/predict.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c4bde9e27f99624d"}},{"code_sha256_prefix":"adb7541f2edf078e","entry":"get_label_list","repo":"dhananjayashok/littlehumandata","repo_kind":"official","path":"llm-utils/classification.py","file_url":"https://github.com/dhananjayashok/littlehumandata/blob/HEAD/llm-utils/classification.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"adb7541f2edf078e"}},{"code_sha256_prefix":"29596155303849f9","entry":"get_metric_report_str","repo":"dhananjayashok/littlehumandata","repo_kind":"official","path":"llm-utils/causal_lm_conditional_generation.py","file_url":"https://github.com/dhananjayashok/littlehumandata/blob/HEAD/llm-utils/causal_lm_conditional_generation.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"29596155303849f9"}},{"code_sha256_prefix":"3a484a15b2187487","entry":"get_metric_report_str","repo":"dhananjayashok/littlehumandata","repo_kind":"official","path":"llm-utils/classification.py","file_url":"https://github.com/dhananjayashok/littlehumandata/blob/HEAD/llm-utils/classification.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3a484a15b2187487"}},{"code_sha256_prefix":"dec1cca2d250dd20","entry":"get_metric_report_str","repo":"dhananjayashok/littlehumandata","repo_kind":"official","path":"llm-utils/common_utils.py","file_url":"https://github.com/dhananjayashok/littlehumandata/blob/HEAD/llm-utils/common_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dec1cca2d250dd20"}},{"code_sha256_prefix":"18bd2e3ef915c4a4","entry":"get_n_gram","repo":"dhananjayashok/littlehumandata","repo_kind":"official","path":"analysis.py","file_url":"https://github.com/dhananjayashok/littlehumandata/blob/HEAD/analysis.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"18bd2e3ef915c4a4"}},{"code_sha256_prefix":"2b29adfbbb6ee76b","entry":"guarded_lower","repo":"dhananjayashok/littlehumandata","repo_kind":"official","path":"add_info_predfile.py","file_url":"https://github.com/dhananjayashok/littlehumandata/blob/HEAD/add_info_predfile.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2b29adfbbb6ee76b"}},{"code_sha256_prefix":"cb42513ddf067d63","entry":"guarded_split","repo":"dhananjayashok/littlehumandata","repo_kind":"official","path":"analysis.py","file_url":"https://github.com/dhananjayashok/littlehumandata/blob/HEAD/analysis.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cb42513ddf067d63"}},{"code_sha256_prefix":"b726f185fd16d31d","entry":"remove_urls","repo":"dhananjayashok/littlehumandata","repo_kind":"official","path":"prompt_generation/prompt_models.py","file_url":"https://github.com/dhananjayashok/littlehumandata/blob/HEAD/prompt_generation/prompt_models.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b726f185fd16d31d"}},{"code_sha256_prefix":"019975ab5eef2a08","entry":"tok_len","repo":"dhananjayashok/littlehumandata","repo_kind":"official","path":"prompt_generation/prompt_models.py","file_url":"https://github.com/dhananjayashok/littlehumandata/blob/HEAD/prompt_generation/prompt_models.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"019975ab5eef2a08"}},{"code_sha256_prefix":"859b0c517f44d1ae","entry":"truncate_context","repo":"dhananjayashok/littlehumandata","repo_kind":"official","path":"prompt_generation/prompt_models.py","file_url":"https://github.com/dhananjayashok/littlehumandata/blob/HEAD/prompt_generation/prompt_models.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"859b0c517f44d1ae"}},{"code_sha256_prefix":"d046034742d3fdba","entry":"compute_exact_match","repo":"dhananjayashok/littlehumandata","repo_kind":"official","path":"compute_qa_metrics.py","file_url":"https://github.com/dhananjayashok/littlehumandata/blob/HEAD/compute_qa_metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d046034742d3fdba"}},{"code_sha256_prefix":"795fca5aa8f88621","entry":"get_model","repo":"dhananjayashok/littlehumandata","repo_kind":"official","path":"llm-utils/predict.py","file_url":"https://github.com/dhananjayashok/littlehumandata/blob/HEAD/llm-utils/predict.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"795fca5aa8f88621"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}