{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/a-few-more-examples-may-be-worth-billions-of","title":"A Few More Examples May Be Worth Billions of Parameters","arxiv_id":"2110.04374","date":"2021-10-08","proceeding":null,"authors":["Yuval Kirstain","Patrick Lewis","Sebastian Riedel","Omer Levy"],"abstract":"We investigate the dynamics of increasing the number of model parameters versus the number of labeled examples across a wide variety of tasks. Our exploration reveals that while scaling parameters consistently yields performance improvements, the contribution of additional examples highly depends on the task's format. Specifically, in open question answering tasks, enlarging the training set does not improve performance. In contrast, classification, extractive question answering, and multiple choice tasks benefit so much from additional examples that collecting a few hundred examples is often \"worth\" billions of parameters. We hypothesize that unlike open question answering, which involves recalling specific information, solving strategies for tasks with a more restricted output space transfer across examples, and can therefore be learned with small amounts of labeled data.","url_abs":"https://arxiv.org/abs/2110.04374v1","url_pdf":"https://arxiv.org/pdf/2110.04374v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"a-few-more-examples-may-be-worth-billions-of","repo_url":"https://github.com/yuvalkirstain/lm-evaluation-harness","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"none","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"extractive-question-answering","task_name":"Extractive Question-Answering"},{"task_slug":"multiple-choice","task_name":"Multiple-choice"},{"task_slug":"open-question","task_name":"Open-Ended Question Answering"},{"task_slug":"question-answering","task_name":"Question Answering"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2110.04374","atlas_url":"https://app.syntology.ai/?focus=2110.04374","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2110.04374"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yuvalkirstain/lm-evaluation-harness","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_violates":2,"ran":6,"ran_draft_wrong":1,"unverified":1},"by_repo_kind":{"official":{"samples":10,"ran":9,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"0928f497e20fb443","entry":"mean","repo":"yuvalkirstain/lm-evaluation-harness","repo_kind":"official","path":"lm_eval/metrics.py","file_url":"https://github.com/yuvalkirstain/lm-evaluation-harness/blob/HEAD/lm_eval/metrics.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":2,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0928f497e20fb443"}},{"code_sha256_prefix":"65ed12119df56f3b","entry":"get_result","repo":"yuvalkirstain/lm-evaluation-harness","repo_kind":"official","path":"lm_eval/models/gpt3.py","file_url":"https://github.com/yuvalkirstain/lm-evaluation-harness/blob/HEAD/lm_eval/models/gpt3.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"65ed12119df56f3b"}},{"code_sha256_prefix":"359c35fdcd7a0ee5","entry":"group","repo":"yuvalkirstain/lm-evaluation-harness","repo_kind":"official","path":"lm_eval/utils.py","file_url":"https://github.com/yuvalkirstain/lm-evaluation-harness/blob/HEAD/lm_eval/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"359c35fdcd7a0ee5"}},{"code_sha256_prefix":"ea06eaae4fc1eaf0","entry":"hash_args","repo":"yuvalkirstain/lm-evaluation-harness","repo_kind":"official","path":"lm_eval/base.py","file_url":"https://github.com/yuvalkirstain/lm-evaluation-harness/blob/HEAD/lm_eval/base.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ea06eaae4fc1eaf0"}},{"code_sha256_prefix":"e18607dae38b30e2","entry":"is_number","repo":"yuvalkirstain/lm-evaluation-harness","repo_kind":"official","path":"lm_eval/utils.py","file_url":"https://github.com/yuvalkirstain/lm-evaluation-harness/blob/HEAD/lm_eval/utils.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e18607dae38b30e2"}},{"code_sha256_prefix":"028eb6dad1493f40","entry":"matthews_corrcoef","repo":"yuvalkirstain/lm-evaluation-harness","repo_kind":"official","path":"lm_eval/metrics.py","file_url":"https://github.com/yuvalkirstain/lm-evaluation-harness/blob/HEAD/lm_eval/metrics.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"028eb6dad1493f40"}},{"code_sha256_prefix":"79a89132ce3f75f1","entry":"median","repo":"yuvalkirstain/lm-evaluation-harness","repo_kind":"official","path":"lm_eval/metrics.py","file_url":"https://github.com/yuvalkirstain/lm-evaluation-harness/blob/HEAD/lm_eval/metrics.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"79a89132ce3f75f1"}},{"code_sha256_prefix":"c6d952282e302ade","entry":"simple_parse_args_string","repo":"yuvalkirstain/lm-evaluation-harness","repo_kind":"official","path":"lm_eval/utils.py","file_url":"https://github.com/yuvalkirstain/lm-evaluation-harness/blob/HEAD/lm_eval/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c6d952282e302ade"}},{"code_sha256_prefix":"31b87866d085212b","entry":"write_datasets_fo_read","repo":"yuvalkirstain/lm-evaluation-harness","repo_kind":"official","path":"lm_eval/train_pl.py","file_url":"https://github.com/yuvalkirstain/lm-evaluation-harness/blob/HEAD/lm_eval/train_pl.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"31b87866d085212b"}},{"code_sha256_prefix":"3b0f3f81457fce8a","entry":"load_raw_datasests","repo":"yuvalkirstain/lm-evaluation-harness","repo_kind":"official","path":"lm_eval/train_pl.py","file_url":"https://github.com/yuvalkirstain/lm-evaluation-harness/blob/HEAD/lm_eval/train_pl.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3b0f3f81457fce8a"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}