{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2508-09093","title":"Scaling Up Active Testing to Large Language Models","arxiv_id":"2508.09093","date":"2025-08-12","proceeding":"NeurIPS","authors":["Gabrielle Berrada","Jannik Kossen","Freddie Bickford Smith","Muhammed Razzak","Yarin Gal","Tom Rainforth"],"abstract":"Active testing enables label-efficient evaluation of predictive models through careful data acquisition, but it can pose a significant computational cost. We identify cost-saving measures that enable active testing to be scaled up to large language models (LLMs). In particular we show that the surrogate model used to guide data acquisition can be constructed cheaply using in-context learning, does not require updating within an active-testing loop, and can be smaller than the target model. We even find we can make good data-acquisition decisions without making predictions with the target model. As a result we are able to achieve much more accurate evaluations of LLM performance relative to using randomly acquired data. We additionally introduce a bootstrap estimator of evaluation error, which we show to be a useful indicator of how well active testing is working within a single run.","url_abs":"https://arxiv.org/abs/2508.09093","url_pdf":"https://arxiv.org/pdf/2508.09093","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2508.09093","atlas_url":"https://app.syntology.ai/?focus=2508.09093","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2508.09093"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/gabrielleberrada/scaling-up-active-testing","reach":{"status":"ok"}}],"summary":{"unverified":7},"by_repo_kind":{"found_in_text":{"samples":7,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":7,"samples":[{"code_sha256_prefix":"c5883bf846a4a3bf","entry":"accuracy","repo":"gabrielleberrada/scaling-up-active-testing","repo_kind":"found_in_text","path":"metrics.py","file_url":"https://github.com/gabrielleberrada/scaling-up-active-testing/blob/HEAD/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c5883bf846a4a3bf"}},{"code_sha256_prefix":"d6b174bff5224831","entry":"get_method_name","repo":"gabrielleberrada/scaling-up-active-testing","repo_kind":"found_in_text","path":"visualize.py","file_url":"https://github.com/gabrielleberrada/scaling-up-active-testing/blob/HEAD/visualize.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d6b174bff5224831"}},{"code_sha256_prefix":"b651cf2d15e43d76","entry":"load_arrays","repo":"gabrielleberrada/scaling-up-active-testing","repo_kind":"found_in_text","path":"utils.py","file_url":"https://github.com/gabrielleberrada/scaling-up-active-testing/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b651cf2d15e43d76"}},{"code_sha256_prefix":"c2edd754e491f0f5","entry":"load_tensors","repo":"gabrielleberrada/scaling-up-active-testing","repo_kind":"found_in_text","path":"utils.py","file_url":"https://github.com/gabrielleberrada/scaling-up-active-testing/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c2edd754e491f0f5"}},{"code_sha256_prefix":"44c95b9e049bea62","entry":"nll","repo":"gabrielleberrada/scaling-up-active-testing","repo_kind":"found_in_text","path":"metrics.py","file_url":"https://github.com/gabrielleberrada/scaling-up-active-testing/blob/HEAD/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"44c95b9e049bea62"}},{"code_sha256_prefix":"9ab2eaf65c6ecef9","entry":"remove_split_layer","repo":"gabrielleberrada/scaling-up-active-testing","repo_kind":"found_in_text","path":"huggingface_model.py","file_url":"https://github.com/gabrielleberrada/scaling-up-active-testing/blob/HEAD/huggingface_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9ab2eaf65c6ecef9"}},{"code_sha256_prefix":"72a98bf997a84cfa","entry":"softmax_loss","repo":"gabrielleberrada/scaling-up-active-testing","repo_kind":"found_in_text","path":"metrics.py","file_url":"https://github.com/gabrielleberrada/scaling-up-active-testing/blob/HEAD/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"72a98bf997a84cfa"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.LG","source":"arxiv_api"},"syntology_extracted_results":null}