{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2601-13885","title":"Confident Rankings with Fewer Items: Adaptive LLM Evaluation with Continuous Scores","arxiv_id":"2601.13885","date":"2026-01-20","proceeding":null,"authors":["Esma Balkır","Alice Pernthaller","Marco Basaldella","José Hernández-Orallo","Nigel Collier"],"abstract":"Computerized Adaptive Testing (CAT) has proven effective for efficient LLM evaluation on multiple-choice benchmarks, but modern LLM evaluation increasingly relies on generation tasks where outputs are scored continuously rather than marked correct/incorrect. We present a principled extension of IRT-based adaptive testing to continuous bounded scores (ROUGE, BLEU, LLM-as-a-Judge) by replacing the Bernoulli response distribution with a heteroskedastic normal distribution. Building on this, we introduce an uncertainty aware ranker with adaptive stopping criteria that achieves reliable model ranking while testing as few items and as cheaply as possible. We validate our method on five benchmarks spanning n-gram-based, embedding-based, and LLM-as-judge metrics. Our method uses 2% of the items while improving ranking correlation by 0.12 τ over random sampling, with 95% accuracy on confident predictions.","url_abs":"https://arxiv.org/abs/2601.13885","url_pdf":"https://arxiv.org/pdf/2601.13885","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2601.13885","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2601.13885"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/trismik/continuous-cat","reach":null}],"summary":{"ran":4,"ran_draft_wrong":4,"ran_honours":4,"ran_violates":1,"unverified":3},"by_repo_kind":{"found_in_text":{"samples":16,"ran":13,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":16,"samples":[{"code_sha256_prefix":"bf2889c619d9777d","entry":"CATSettings","repo":"trismik/continuous-cat","repo_kind":"found_in_text","path":"src/continuous_cat/ranker.py","file_url":"https://github.com/trismik/continuous-cat/blob/HEAD/src/continuous_cat/ranker.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"bf2889c619d9777d"}},{"code_sha256_prefix":"e0814bc95a76e031","entry":"CATState","repo":"trismik/continuous-cat","repo_kind":"found_in_text","path":"src/continuous_cat/ranker.py","file_url":"https://github.com/trismik/continuous-cat/blob/HEAD/src/continuous_cat/ranker.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"e0814bc95a76e031"}},{"code_sha256_prefix":"088c7b5eb9ed7fc1","entry":"Item","repo":"trismik/continuous-cat","repo_kind":"found_in_text","path":"src/continuous_cat/ranker.py","file_url":"https://github.com/trismik/continuous-cat/blob/HEAD/src/continuous_cat/ranker.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"088c7b5eb9ed7fc1"}},{"code_sha256_prefix":"ae213b4890bfed90","entry":"RankingResult","repo":"trismik/continuous-cat","repo_kind":"found_in_text","path":"src/continuous_cat/ranker.py","file_url":"https://github.com/trismik/continuous-cat/blob/HEAD/src/continuous_cat/ranker.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"ae213b4890bfed90"}},{"code_sha256_prefix":"306e0d8065457af7","entry":"bayesian_update","repo":"trismik/continuous-cat","repo_kind":"found_in_text","path":"src/continuous_cat/ranker.py","file_url":"https://github.com/trismik/continuous-cat/blob/HEAD/src/continuous_cat/ranker.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"306e0d8065457af7"}},{"code_sha256_prefix":"b889a21451419e0c","entry":"compute_pairwise_probability","repo":"trismik/continuous-cat","repo_kind":"found_in_text","path":"src/continuous_cat/ranker.py","file_url":"https://github.com/trismik/continuous-cat/blob/HEAD/src/continuous_cat/ranker.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"b889a21451419e0c"}},{"code_sha256_prefix":"e7ec7205a622148d","entry":"estimate_avg_item_cost","repo":"trismik/continuous-cat","repo_kind":"found_in_text","path":"src/continuous_cat/ranker.py","file_url":"https://github.com/trismik/continuous-cat/blob/HEAD/src/continuous_cat/ranker.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"e7ec7205a622148d"}},{"code_sha256_prefix":"5ef9021490742780","entry":"fisher_info","repo":"trismik/continuous-cat","repo_kind":"found_in_text","path":"src/continuous_cat/ranker.py","file_url":"https://github.com/trismik/continuous-cat/blob/HEAD/src/continuous_cat/ranker.py","link_basis":"first_harvest_node","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"5ef9021490742780"}},{"code_sha256_prefix":"b638bcd2daf86aab","entry":"get_current_ranking","repo":"trismik/continuous-cat","repo_kind":"found_in_text","path":"src/continuous_cat/ranker.py","file_url":"https://github.com/trismik/continuous-cat/blob/HEAD/src/continuous_cat/ranker.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"b638bcd2daf86aab"}},{"code_sha256_prefix":"aba1885f3a6b71f2","entry":"get_uncertain_models","repo":"trismik/continuous-cat","repo_kind":"found_in_text","path":"src/continuous_cat/ranker.py","file_url":"https://github.com/trismik/continuous-cat/blob/HEAD/src/continuous_cat/ranker.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"aba1885f3a6b71f2"}},{"code_sha256_prefix":"176e6a17334b1190","entry":"initialize_theta","repo":"trismik/continuous-cat","repo_kind":"found_in_text","path":"src/continuous_cat/ranker.py","file_url":"https://github.com/trismik/continuous-cat/blob/HEAD/src/continuous_cat/ranker.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"176e6a17334b1190"}},{"code_sha256_prefix":"78bccdbf5e2e2b21","entry":"logistic","repo":"trismik/continuous-cat","repo_kind":"found_in_text","path":"src/continuous_cat/ranker.py","file_url":"https://github.com/trismik/continuous-cat/blob/HEAD/src/continuous_cat/ranker.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"78bccdbf5e2e2b21"}},{"code_sha256_prefix":"9c4a88116b2b2b4e","entry":"select_item_mfi","repo":"trismik/continuous-cat","repo_kind":"found_in_text","path":"src/continuous_cat/ranker.py","file_url":"https://github.com/trismik/continuous-cat/blob/HEAD/src/continuous_cat/ranker.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"9c4a88116b2b2b4e"}},{"code_sha256_prefix":"a8f75622f01fc4d0","entry":"add_costs_to_items","repo":"trismik/continuous-cat","repo_kind":"found_in_text","path":"src/continuous_cat/ranker.py","file_url":"https://github.com/trismik/continuous-cat/blob/HEAD/src/continuous_cat/ranker.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"a8f75622f01fc4d0"}},{"code_sha256_prefix":"64885239ca65a0fd","entry":"compute_item_cost","repo":"trismik/continuous-cat","repo_kind":"found_in_text","path":"src/continuous_cat/ranker.py","file_url":"https://github.com/trismik/continuous-cat/blob/HEAD/src/continuous_cat/ranker.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"64885239ca65a0fd"}},{"code_sha256_prefix":"447f2b7dca3a8ce5","entry":"rank_models","repo":"trismik/continuous-cat","repo_kind":"found_in_text","path":"src/continuous_cat/ranker.py","file_url":"https://github.com/trismik/continuous-cat/blob/HEAD/src/continuous_cat/ranker.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"447f2b7dca3a8ce5"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.CL","source":"arxiv_2026.jsonl"},"syntology_extracted_results":null}