{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2602-02589","title":"PeerRank: Autonomous LLM Evaluation Through Web-Grounded, Bias-Controlled Peer Review","arxiv_id":"2602.02589","date":"2026-02-01","proceeding":null,"authors":["Yanki Margalit","Erni Avram","Ran Taig","Oded Margalit","Nurit Cohen-Inger"],"abstract":"Evaluating large language models typically relies on human-authored benchmarks, reference answers, and human or single-model judgments, approaches that scale poorly, become quickly outdated, and mismatch open-world deployments that depend on web retrieval and synthesis. We introduce PeerRank, a fully autonomous end-to-end evaluation framework in which models generate evaluation tasks, answer them with category-scoped live web grounding, judge peer responses and aggregate dense peer assessments into relative performance estimates, without human supervision or gold references. PeerRank treats evaluation as a multi-agent process where each model participates symmetrically as task designer, respondent, and evaluator, while removing biased judgments. In a large-scale study over 12 commercially available models and 420 autonomously generated questions, PeerRank produces stable, discriminative rankings and reveals measurable identity and presentation biases. Rankings are robust, and mean peer scores agree with Elo. We further validate PeerRank on TruthfulQA and GSM8K, where peer scores correlate with objective accuracy. Together, these results suggest that bias-aware peer evaluation with selective web-grounded answering can scale open-world LLM assessment beyond static and human curated benchmarks.","url_abs":"https://arxiv.org/abs/2602.02589","url_pdf":"https://arxiv.org/pdf/2602.02589","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2602.02589","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2602.02589"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/caura-ai/caura-PeerRank","reach":null}],"summary":{"unverified":11},"by_repo_kind":{"found_in_text":{"samples":11,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"76b9fd5321ad4c7c","entry":"get_color","repo":"caura-ai/caura-PeerRank","repo_kind":"found_in_text","path":"peerrank/figure_utils.py","file_url":"https://github.com/caura-ai/caura-PeerRank/blob/HEAD/peerrank/figure_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"76b9fd5321ad4c7c"}},{"code_sha256_prefix":"044fec136989f0d2","entry":"get_short_name","repo":"caura-ai/caura-PeerRank","repo_kind":"found_in_text","path":"peerrank/config.py","file_url":"https://github.com/caura-ai/caura-PeerRank/blob/HEAD/peerrank/config.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"044fec136989f0d2"}},{"code_sha256_prefix":"41cdaee442f2d812","entry":"get_text_color_for_background","repo":"caura-ai/caura-PeerRank","repo_kind":"found_in_text","path":"peerrank/figure_utils.py","file_url":"https://github.com/caura-ai/caura-PeerRank/blob/HEAD/peerrank/figure_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"41cdaee442f2d812"}},{"code_sha256_prefix":"651f5dab5cafde82","entry":"get_timing_data","repo":"caura-ai/caura-PeerRank","repo_kind":"found_in_text","path":"generate_figures_PeerRank.py","file_url":"https://github.com/caura-ai/caura-PeerRank/blob/HEAD/generate_figures_PeerRank.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"651f5dab5cafde82"}},{"code_sha256_prefix":"8606cbbc71dbca7d","entry":"load_data","repo":"caura-ai/caura-PeerRank","repo_kind":"found_in_text","path":"generate_figures_PeerRank.py","file_url":"https://github.com/caura-ai/caura-PeerRank/blob/HEAD/generate_figures_PeerRank.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8606cbbc71dbca7d"}},{"code_sha256_prefix":"b0e93efc0814802c","entry":"remap_scores_to_models","repo":"caura-ai/caura-PeerRank","repo_kind":"found_in_text","path":"peerrank_phase3.py","file_url":"https://github.com/caura-ai/caura-PeerRank/blob/HEAD/peerrank_phase3.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b0e93efc0814802c"}},{"code_sha256_prefix":"67312eaf38e60f5c","entry":"remap_scores_to_models","repo":"caura-ai/caura-PeerRank","repo_kind":"found_in_text","path":"peerrank_ui.py","file_url":"https://github.com/caura-ai/caura-PeerRank/blob/HEAD/peerrank_ui.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"67312eaf38e60f5c"}},{"code_sha256_prefix":"2cdfd127e45b5201","entry":"sanitize_prompt","repo":"caura-ai/caura-PeerRank","repo_kind":"found_in_text","path":"peerrank/providers.py","file_url":"https://github.com/caura-ai/caura-PeerRank/blob/HEAD/peerrank/providers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2cdfd127e45b5201"}},{"code_sha256_prefix":"e697af0f40c4bee3","entry":"serpapi_search","repo":"caura-ai/caura-PeerRank","repo_kind":"found_in_text","path":"peerrank/providers.py","file_url":"https://github.com/caura-ai/caura-PeerRank/blob/HEAD/peerrank/providers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e697af0f40c4bee3"}},{"code_sha256_prefix":"c91159eb1f2e6405","entry":"set_active_categories","repo":"caura-ai/caura-PeerRank","repo_kind":"found_in_text","path":"peerrank/config.py","file_url":"https://github.com/caura-ai/caura-PeerRank/blob/HEAD/peerrank/config.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c91159eb1f2e6405"}},{"code_sha256_prefix":"b5132ba6b51d4482","entry":"tavily_search","repo":"caura-ai/caura-PeerRank","repo_kind":"found_in_text","path":"peerrank/providers.py","file_url":"https://github.com/caura-ai/caura-PeerRank/blob/HEAD/peerrank/providers.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b5132ba6b51d4482"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.LG","source":"arxiv_2026.jsonl"},"syntology_extracted_results":null}