{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/routereval-a-comprehensive-benchmark-for","title":"RouterEval: A Comprehensive Benchmark for Routing LLMs to Explore Model-level Scaling Up in LLMs","arxiv_id":"2503.10657","date":"2025-03-08","proceeding":null,"authors":["Zhongzhan Huang","Guoming Ling","Yupei Lin","Yandong Chen","Shanshan Zhong","Hefeng Wu","Liang Lin"],"abstract":"Routing large language models (LLMs) is a new paradigm that uses a router to recommend the best LLM from a pool of candidates for a given input. In this paper, our comprehensive analysis with more than 8,500 LLMs reveals a novel model-level scaling up phenomenon in Routing LLMs, i.e., a capable router can significantly enhance the performance of this paradigm as the number of candidates increases. This improvement can even surpass the performance of the best single model in the pool and many existing strong LLMs, confirming it a highly promising paradigm. However, the lack of comprehensive and open-source benchmarks for Routing LLMs has hindered the development of routers. In this paper, we introduce RouterEval, a benchmark tailored for router research, which includes over 200,000,000 performance records for 12 popular LLM evaluations across various areas such as commonsense reasoning, semantic understanding, etc., based on over 8,500 various LLMs. Using RouterEval, extensive evaluations of existing Routing LLM methods reveal that most still have significant room for improvement. See https://github.com/MilkThink-Lab/RouterEval for all data, code and tutorial.","url_abs":"https://arxiv.org/abs/2503.10657v2","url_pdf":"https://arxiv.org/pdf/2503.10657v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"routereval-a-comprehensive-benchmark-for","repo_url":"https://github.com/milkthink-lab/routereval","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"instruction-following","task_name":"Instruction Following"},{"task_slug":"mathematical-reasoning","task_name":"Mathematical Reasoning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2503.10657","atlas_url":"https://app.syntology.ai/?focus=2503.10657","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2503.10657"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/milkthink-lab/routereval","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":12,"unverified":1},"by_repo_kind":{"official":{"samples":13,"ran":12,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"deff5e9b4c4a78eb","entry":"a3m_ensemble_route","repo":"milkthink-lab/routereval","repo_kind":"official","path":"router/A3M-router/a3m_router.py","file_url":"https://github.com/milkthink-lab/routereval/blob/HEAD/router/A3M-router/a3m_router.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"deff5e9b4c4a78eb"}},{"code_sha256_prefix":"2713e130185a23e5","entry":"constant","repo":"milkthink-lab/routereval","repo_kind":"official","path":"router/MLPR_LinearR/tools.py","file_url":"https://github.com/milkthink-lab/routereval/blob/HEAD/router/MLPR_LinearR/tools.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2713e130185a23e5"}},{"code_sha256_prefix":"7529a26792fd6da5","entry":"cosine_lr","repo":"milkthink-lab/routereval","repo_kind":"official","path":"router/MLPR_LinearR/tools.py","file_url":"https://github.com/milkthink-lab/routereval/blob/HEAD/router/MLPR_LinearR/tools.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7529a26792fd6da5"}},{"code_sha256_prefix":"6bc2f9a20da48d95","entry":"create_prompts","repo":"milkthink-lab/routereval","repo_kind":"official","path":"utils.py","file_url":"https://github.com/milkthink-lab/routereval/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6bc2f9a20da48d95"}},{"code_sha256_prefix":"fd82a32520c85a92","entry":"dataset_perpare","repo":"milkthink-lab/routereval","repo_kind":"official","path":"router/RoBERTa-MLC/roberta_MLC.py","file_url":"https://github.com/milkthink-lab/routereval/blob/HEAD/router/RoBERTa-MLC/roberta_MLC.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fd82a32520c85a92"}},{"code_sha256_prefix":"50768d80fbec340b","entry":"mix_router","repo":"milkthink-lab/routereval","repo_kind":"official","path":"router/R_o/r_o_router.py","file_url":"https://github.com/milkthink-lab/routereval/blob/HEAD/router/R_o/r_o_router.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"50768d80fbec340b"}},{"code_sha256_prefix":"9a63a1d713d235f6","entry":"oracle_router","repo":"milkthink-lab/routereval","repo_kind":"official","path":"get_router_dataset.py","file_url":"https://github.com/milkthink-lab/routereval/blob/HEAD/get_router_dataset.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9a63a1d713d235f6"}},{"code_sha256_prefix":"1f7e67d5b3462795","entry":"predict_proba","repo":"milkthink-lab/routereval","repo_kind":"official","path":"router/C-RoBERTa-cluster/cluster.py","file_url":"https://github.com/milkthink-lab/routereval/blob/HEAD/router/C-RoBERTa-cluster/cluster.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1f7e67d5b3462795"}},{"code_sha256_prefix":"7c51505a5adb6750","entry":"prepare_data","repo":"milkthink-lab/routereval","repo_kind":"official","path":"utils.py","file_url":"https://github.com/milkthink-lab/routereval/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7c51505a5adb6750"}},{"code_sha256_prefix":"80695e8696cdbc52","entry":"process_label","repo":"milkthink-lab/routereval","repo_kind":"official","path":"get_router_dataset.py","file_url":"https://github.com/milkthink-lab/routereval/blob/HEAD/get_router_dataset.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"80695e8696cdbc52"}},{"code_sha256_prefix":"d704eae539e693d2","entry":"split_data","repo":"milkthink-lab/routereval","repo_kind":"official","path":"get_router_dataset.py","file_url":"https://github.com/milkthink-lab/routereval/blob/HEAD/get_router_dataset.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d704eae539e693d2"}},{"code_sha256_prefix":"3ea5687c1026fe43","entry":"step_lr","repo":"milkthink-lab/routereval","repo_kind":"official","path":"router/MLPR_LinearR/tools.py","file_url":"https://github.com/milkthink-lab/routereval/blob/HEAD/router/MLPR_LinearR/tools.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3ea5687c1026fe43"}},{"code_sha256_prefix":"8e6400b1962406a9","entry":"create_embeds","repo":"milkthink-lab/routereval","repo_kind":"official","path":"utils.py","file_url":"https://github.com/milkthink-lab/routereval/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8e6400b1962406a9"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}