{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2605-01630","title":"Prosa: Rubric-Based Evaluation of LLMs on Real User Chats in Brazilian Portuguese","arxiv_id":"2605.01630","date":"2026-05-02","proceeding":null,"authors":["Roseval Malaquias Junior","Giovana Kerche Bonás","Thales Sales Almeida","Hugo Abonizio","Thiago Laitz","Ramon Pires","Marcos Piau","Celio Larcher","Rodrigo Nogueira"],"abstract":"Rankings produced by holistic LLM-as-a-judge scoring are sensitive to the bias of the chosen judge model. We show that switching to binary rubric scoring with multi-judge filtering removes this sensitivity: decomposing the judgement matters more than the judge model itself. To support this claim, we introduce Prosa, the first real user multi-turn Brazilian Portuguese chat benchmark: 1,000 WildChat conversations scored by three judges from three model families on 16 models. Under filtered rubric scoring the three judges agree on every one of the 16 ranks, whereas under holistic scoring they agree on only 7 of 16. Additionally, the rubric filtering pipeline increases the average score gap between neighbouring models by 47%, thereby improving Prosa's discriminative power. Evaluating a new model on Prosa costs approximately $2.1 when using Gemini 3 Flash as the judge. We release the benchmark and the filtering code to ensure that future models can be assessed under identical conditions. These artifacts also make our rubric-based scoring method reusable beyond Prosa, supporting other open-ended evaluation settings.","url_abs":"https://arxiv.org/abs/2605.01630","url_pdf":"https://arxiv.org/pdf/2605.01630","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2605.01630","atlas_url":"https://app.syntology.ai/?focus=2605.01630","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2605.01630"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/maritaca-ai/Prosa","reach":null}],"summary":{"ran_draft_wrong":5,"unverified":1},"by_repo_kind":{"found_in_text":{"samples":6,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":6,"samples":[{"code_sha256_prefix":"dafb81be1aa7ce06","entry":"build_score_rubric_prompt","repo":"maritaca-ai/Prosa","repo_kind":"found_in_text","path":"Prosa-benchmark/prosa/gen_score.py","file_url":"https://github.com/maritaca-ai/Prosa/blob/HEAD/Prosa-benchmark/prosa/gen_score.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"dafb81be1aa7ce06"}},{"code_sha256_prefix":"a258f3188fb6b6e9","entry":"compute_score","repo":"maritaca-ai/Prosa","repo_kind":"found_in_text","path":"Prosa-benchmark/prosa/gen_score.py","file_url":"https://github.com/maritaca-ai/Prosa/blob/HEAD/Prosa-benchmark/prosa/gen_score.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a258f3188fb6b6e9"}},{"code_sha256_prefix":"62055c6ce37e2c52","entry":"get_model_response","repo":"maritaca-ai/Prosa","repo_kind":"found_in_text","path":"Prosa-benchmark/prosa/gen_score.py","file_url":"https://github.com/maritaca-ai/Prosa/blob/HEAD/Prosa-benchmark/prosa/gen_score.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"62055c6ce37e2c52"}},{"code_sha256_prefix":"bf757c208e4ef0d0","entry":"parse_checklist","repo":"maritaca-ai/Prosa","repo_kind":"found_in_text","path":"Prosa-benchmark/prosa/gen_score.py","file_url":"https://github.com/maritaca-ai/Prosa/blob/HEAD/Prosa-benchmark/prosa/gen_score.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bf757c208e4ef0d0"}},{"code_sha256_prefix":"a4ccd2fd90d3be94","entry":"parse_evaluations","repo":"maritaca-ai/Prosa","repo_kind":"found_in_text","path":"Prosa-benchmark/prosa/gen_score.py","file_url":"https://github.com/maritaca-ai/Prosa/blob/HEAD/Prosa-benchmark/prosa/gen_score.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a4ccd2fd90d3be94"}},{"code_sha256_prefix":"54f89c8c0e882854","entry":"evaluate_single_rrd","repo":"maritaca-ai/Prosa","repo_kind":"found_in_text","path":"Prosa-benchmark/prosa/gen_score.py","file_url":"https://github.com/maritaca-ai/Prosa/blob/HEAD/Prosa-benchmark/prosa/gen_score.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"54f89c8c0e882854"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.CL","source":"arxiv_2026.jsonl"},"syntology_extracted_results":null}