{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/judging-the-judges-a-systematic-investigation","title":"Judging the Judges: A Systematic Study of Position Bias in LLM-as-a-Judge","arxiv_id":"2406.07791","date":"2024-06-12","proceeding":null,"authors":["Lin Shi","Chiyu Ma","Wenhua Liang","Xingjian Diao","Weicheng Ma","Soroush Vosoughi"],"abstract":"LLM-as-a-Judge has emerged as a promising alternative to human evaluators across various tasks, yet inherent biases - particularly position bias, the tendency to favor solutions based on their position within the prompt - compromise its reliability. This exploratory study evaluates position bias in LLM judges across pairwise and list-wise comparison settings, introducing three metrics: repetition stability, position consistency, and preference fairness. Our experiments, involving 15 LLM judges across MTBench and DevBench with 22 tasks and approximately 40 solution-generating models, result in over 150,000 evaluation instances. We identify Judge-Level, Candidate-Level, and Task-Level factors contributing to bias. The findings confirm that position bias is not due to random chance and varies significantly across judges and tasks. While position bias is weakly influenced by the length of prompt components, it is strongly affected by the quality gap between solutions. Our agreement and disagreement analysis among judges further provides insights into the distribution of judging difficulty across the dataset, and highlights the potential for dataset modifications.","url_abs":"https://arxiv.org/abs/2406.07791v8","url_pdf":"https://arxiv.org/pdf/2406.07791v8.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"judging-the-judges-a-systematic-investigation","repo_url":"https://github.com/Slimshilin/Position-Bias-Analyzer-Demo","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}}],"tasks":[{"task_slug":"fairness","task_name":"Fairness"},{"task_slug":null,"task_name":"Position"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt-4","method_name":"GPT-4"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2406.07791","atlas_url":"https://app.syntology.ai/?focus=2406.07791","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.07791"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Slimshilin/Position-Bias-Analyzer-Demo","reach":{"status":"ok"}}],"summary":{"ran":16},"by_repo_kind":{"official":{"samples":16,"ran":16,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":16,"samples":[{"code_sha256_prefix":"b3186cda66ba698e","entry":"Claude_context_window","repo":"Slimshilin/Position-Bias-Analyzer-Demo","repo_kind":"official","path":"subeval/chat_api/claude.py","file_url":"https://github.com/Slimshilin/Position-Bias-Analyzer-Demo/blob/HEAD/subeval/chat_api/claude.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b3186cda66ba698e"}},{"code_sha256_prefix":"acd86dcb3b21fb02","entry":"GPT_context_window","repo":"Slimshilin/Position-Bias-Analyzer-Demo","repo_kind":"official","path":"subeval/chat_api/gpt.py","file_url":"https://github.com/Slimshilin/Position-Bias-Analyzer-Demo/blob/HEAD/subeval/chat_api/gpt.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"acd86dcb3b21fb02"}},{"code_sha256_prefix":"4edd4f61d223fe52","entry":"Gemini_context_window","repo":"Slimshilin/Position-Bias-Analyzer-Demo","repo_kind":"official","path":"subeval/chat_api/gemini.py","file_url":"https://github.com/Slimshilin/Position-Bias-Analyzer-Demo/blob/HEAD/subeval/chat_api/gemini.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4edd4f61d223fe52"}},{"code_sha256_prefix":"6a696ae76c55b5e2","entry":"aggregate_judge_data","repo":"Slimshilin/Position-Bias-Analyzer-Demo","repo_kind":"official","path":"position_bias_analyzer/calculate_consistency_and_preference/consistency_preference_util.py","file_url":"https://github.com/Slimshilin/Position-Bias-Analyzer-Demo/blob/HEAD/position_bias_analyzer/calculate_consistency_and_preference/consistency_preference_util.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6a696ae76c55b5e2"}},{"code_sha256_prefix":"bffbd3fb4e1ab9a6","entry":"augment_length_stats","repo":"Slimshilin/Position-Bias-Analyzer-Demo","repo_kind":"official","path":"position_bias_analyzer/position_bias_vs_factors/position_bias_vs_lengths/calculate_length_stats.py","file_url":"https://github.com/Slimshilin/Position-Bias-Analyzer-Demo/blob/HEAD/position_bias_analyzer/position_bias_vs_factors/position_bias_vs_lengths/calculate_length_stats.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bffbd3fb4e1ab9a6"}},{"code_sha256_prefix":"ff69251cef4fa95a","entry":"augment_results_with_overall_winrates","repo":"Slimshilin/Position-Bias-Analyzer-Demo","repo_kind":"official","path":"position_bias_analyzer/position_bias_vs_factors/position_bias_vs_winrates/calculate_overall_winrate.py","file_url":"https://github.com/Slimshilin/Position-Bias-Analyzer-Demo/blob/HEAD/position_bias_analyzer/position_bias_vs_factors/position_bias_vs_winrates/calculate_overall_winrate.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ff69251cef4fa95a"}},{"code_sha256_prefix":"390a970b1fdce16c","entry":"calculate_length_stats","repo":"Slimshilin/Position-Bias-Analyzer-Demo","repo_kind":"official","path":"position_bias_analyzer/position_bias_vs_factors/position_bias_vs_lengths/calculate_length_stats.py","file_url":"https://github.com/Slimshilin/Position-Bias-Analyzer-Demo/blob/HEAD/position_bias_analyzer/position_bias_vs_factors/position_bias_vs_lengths/calculate_length_stats.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"390a970b1fdce16c"}},{"code_sha256_prefix":"1ec9bc4d7069a870","entry":"calculate_overall_winrate","repo":"Slimshilin/Position-Bias-Analyzer-Demo","repo_kind":"official","path":"position_bias_analyzer/position_bias_vs_factors/position_bias_vs_winrates/calculate_overall_winrate.py","file_url":"https://github.com/Slimshilin/Position-Bias-Analyzer-Demo/blob/HEAD/position_bias_analyzer/position_bias_vs_factors/position_bias_vs_winrates/calculate_overall_winrate.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1ec9bc4d7069a870"}},{"code_sha256_prefix":"2a367c5cad3f8101","entry":"calculate_positional_consistency_and_preference_score","repo":"Slimshilin/Position-Bias-Analyzer-Demo","repo_kind":"official","path":"position_bias_analyzer/calculate_consistency_and_preference/consistency_preference_calculation.py","file_url":"https://github.com/Slimshilin/Position-Bias-Analyzer-Demo/blob/HEAD/position_bias_analyzer/calculate_consistency_and_preference/consistency_preference_calculation.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2a367c5cad3f8101"}},{"code_sha256_prefix":"e8277b4e02292cbe","entry":"cn_string","repo":"Slimshilin/Position-Bias-Analyzer-Demo","repo_kind":"official","path":"subeval/smp.py","file_url":"https://github.com/Slimshilin/Position-Bias-Analyzer-Demo/blob/HEAD/subeval/smp.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e8277b4e02292cbe"}},{"code_sha256_prefix":"726da9cc7c6c575a","entry":"extract_answer","repo":"Slimshilin/Position-Bias-Analyzer-Demo","repo_kind":"official","path":"position_bias_analyzer/calculate_consistency_and_preference/consistency_preference_util.py","file_url":"https://github.com/Slimshilin/Position-Bias-Analyzer-Demo/blob/HEAD/position_bias_analyzer/calculate_consistency_and_preference/consistency_preference_util.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"726da9cc7c6c575a"}},{"code_sha256_prefix":"ecf4b23852cb6c61","entry":"match_answer","repo":"Slimshilin/Position-Bias-Analyzer-Demo","repo_kind":"official","path":"position_bias_analyzer/calculate_consistency_and_preference/consistency_preference_util.py","file_url":"https://github.com/Slimshilin/Position-Bias-Analyzer-Demo/blob/HEAD/position_bias_analyzer/calculate_consistency_and_preference/consistency_preference_util.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ecf4b23852cb6c61"}},{"code_sha256_prefix":"1f3d4cff9e4b57fd","entry":"match_answer","repo":"Slimshilin/Position-Bias-Analyzer-Demo","repo_kind":"official","path":"subeval/subjective/analyze_util.py","file_url":"https://github.com/Slimshilin/Position-Bias-Analyzer-Demo/blob/HEAD/subeval/subjective/analyze_util.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1f3d4cff9e4b57fd"}},{"code_sha256_prefix":"9f7a492d259f95bd","entry":"proc_task","repo":"Slimshilin/Position-Bias-Analyzer-Demo","repo_kind":"official","path":"subeval/subjective/analyze_util.py","file_url":"https://github.com/Slimshilin/Position-Bias-Analyzer-Demo/blob/HEAD/subeval/subjective/analyze_util.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9f7a492d259f95bd"}},{"code_sha256_prefix":"046fc285c89d080d","entry":"splitlen","repo":"Slimshilin/Position-Bias-Analyzer-Demo","repo_kind":"official","path":"subeval/smp.py","file_url":"https://github.com/Slimshilin/Position-Bias-Analyzer-Demo/blob/HEAD/subeval/smp.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"046fc285c89d080d"}},{"code_sha256_prefix":"b893e74b04f04876","entry":"timestr","repo":"Slimshilin/Position-Bias-Analyzer-Demo","repo_kind":"official","path":"subeval/smp.py","file_url":"https://github.com/Slimshilin/Position-Bias-Analyzer-Demo/blob/HEAD/subeval/smp.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b893e74b04f04876"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}