{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/debatable-intelligence-benchmarking-llm","title":"Debatable Intelligence: Benchmarking LLM Judges via Debate Speech Evaluation","arxiv_id":"2506.05062","date":"2025-06-05","proceeding":null,"authors":["Noy Sternlicht","Ariel Gera","Roy Bar-Haim","Tom Hope","Noam Slonim"],"abstract":"We introduce Debate Speech Evaluation as a novel and challenging benchmark for assessing LLM judges. Evaluating debate speeches requires a deep understanding of the speech at multiple levels, including argument strength and relevance, the coherence and organization of the speech, the appropriateness of its style and tone, and so on. This task involves a unique set of cognitive abilities that have previously received limited attention in systematic LLM benchmarking. To explore such skills, we leverage a dataset of over 600 meticulously annotated debate speeches and present the first in-depth analysis of how state-of-the-art LLMs compare to human judges on this task. Our findings reveal a nuanced picture: while larger models can approximate individual human judgments in some respects, they differ substantially in their overall judgment behavior. We also investigate the ability of frontier LLMs to generate persuasive, opinionated speeches, showing that models may perform at a human level on this task.","url_abs":"https://arxiv.org/abs/2506.05062v1","url_pdf":"https://arxiv.org/pdf/2506.05062v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"debatable-intelligence-benchmarking-llm","repo_url":"https://github.com/noy-sternlicht/debatable-intelligence","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":null}],"tasks":[{"task_slug":"benchmarking","task_name":"Benchmarking"}],"methods":[{"method_slug":"attention","method_name":"Attention"},{"method_slug":"set","method_name":"SET"},{"method_slug":"softmax","method_name":"Softmax"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2506.05062","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2506.05062"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/noy-sternlicht/debatable-intelligence","reach":null}],"summary":{"ran_honours":1,"ran_draft_wrong":2},"by_repo_kind":{"official":{"samples":3,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":3,"samples":[{"code_sha256_prefix":"240d05f321a36efe","entry":"avg_all_annotators","repo":"noy-sternlicht/debatable-intelligence","repo_kind":"official","path":"src/analyse_agreement_with_mean.py","file_url":"https://github.com/noy-sternlicht/debatable-intelligence/blob/HEAD/src/analyse_agreement_with_mean.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"240d05f321a36efe"}},{"code_sha256_prefix":"e94375a7e42a34df","entry":"get_annotations_per_human_annotators_pair","repo":"noy-sternlicht/debatable-intelligence","repo_kind":"official","path":"src/analyse_pairwise_agreement.py","file_url":"https://github.com/noy-sternlicht/debatable-intelligence/blob/HEAD/src/analyse_pairwise_agreement.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e94375a7e42a34df"}},{"code_sha256_prefix":"6215efe708be21ee","entry":"get_annotations_per_human_model_pair","repo":"noy-sternlicht/debatable-intelligence","repo_kind":"official","path":"src/analyse_pairwise_agreement.py","file_url":"https://github.com/noy-sternlicht/debatable-intelligence/blob/HEAD/src/analyse_pairwise_agreement.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"none","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6215efe708be21ee"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}