{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/mario-eval-evaluate-your-math-llm-with-your","title":"MARIO Eval: Evaluate Your Math LLM with your Math LLM--A mathematical dataset evaluation toolkit","arxiv_id":"2404.13925","date":"2024-04-22","proceeding":null,"authors":["Boning Zhang","Chengxi Li","Kai Fan"],"abstract":"Large language models (LLMs) have been explored in a variety of reasoning tasks including solving of mathematical problems. Each math dataset typically includes its own specially designed evaluation script, which, while suitable for its intended use, lacks generalizability across different datasets. Consequently, updates and adaptations to these evaluation tools tend to occur without being systematically reported, leading to inconsistencies and obstacles to fair comparison across studies. To bridge this gap, we introduce a comprehensive mathematical evaluation toolkit that not only utilizes a python computer algebra system (CAS) for its numerical accuracy, but also integrates an optional LLM, known for its considerable natural language processing capabilities. To validate the effectiveness of our toolkit, we manually annotated two distinct datasets. Our experiments demonstrate that the toolkit yields more robust evaluation results compared to prior works, even without an LLM. Furthermore, when an LLM is incorporated, there is a notable enhancement. The code for our method will be made available at \\url{https://github.com/MARIO-Math-Reasoning/math_evaluation}.","url_abs":"https://arxiv.org/abs/2404.13925v1","url_pdf":"https://arxiv.org/pdf/2404.13925v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"mario-eval-evaluate-your-math-llm-with-your","repo_url":"https://github.com/mario-math-reasoning/math_evaluation","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"mario-eval-evaluate-your-math-llm-with-your","repo_url":"https://github.com/MARIO-Math-Reasoning/Super_MARIO","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"mario-eval-evaluate-your-math-llm-with-your","repo_url":"https://github.com/mario-math-reasoning/mario_eval","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":null},{"paper_slug":"mario-eval-evaluate-your-math-llm-with-your","repo_url":"https://github.com/microsoft/rstar","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"math","task_name":"Math"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2404.13925","atlas_url":"https://app.syntology.ai/?focus=2404.13925","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2404.13925"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/MARIO-Math-Reasoning/Super_MARIO","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/microsoft/rstar","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mario-math-reasoning/math_evaluation","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mario-math-reasoning/mario_eval","reach":null}],"summary":{"ran":8,"unverified":5},"by_repo_kind":{"official":{"samples":10,"ran":8,"repositories":1},"listed":{"samples":3,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":13,"samples":[{"code_sha256_prefix":"1b7529dcbd572ad3","entry":"call_openai","repo":"mario-math-reasoning/math_evaluation","repo_kind":"official","path":"math_evaluation/llms/llm_models.py","file_url":"https://github.com/mario-math-reasoning/math_evaluation/blob/HEAD/math_evaluation/llms/llm_models.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1b7529dcbd572ad3"}},{"code_sha256_prefix":"e26a3400c71ced27","entry":"call_tongyi","repo":"mario-math-reasoning/math_evaluation","repo_kind":"official","path":"math_evaluation/llms/llm_models.py","file_url":"https://github.com/mario-math-reasoning/math_evaluation/blob/HEAD/math_evaluation/llms/llm_models.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e26a3400c71ced27"}},{"code_sha256_prefix":"9abaf0f0a3821743","entry":"create_llm","repo":"mario-math-reasoning/math_evaluation","repo_kind":"official","path":"math_evaluation/llms/llm_models.py","file_url":"https://github.com/mario-math-reasoning/math_evaluation/blob/HEAD/math_evaluation/llms/llm_models.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9abaf0f0a3821743"}},{"code_sha256_prefix":"9815785426c107dd","entry":"extract_boxed","repo":"mario-math-reasoning/math_evaluation","repo_kind":"official","path":"example_tora_eval.py","file_url":"https://github.com/mario-math-reasoning/math_evaluation/blob/HEAD/example_tora_eval.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9815785426c107dd"}},{"code_sha256_prefix":"08baa10ca8bb0932","entry":"float_to_latex_scientific","repo":"mario-math-reasoning/math_evaluation","repo_kind":"official","path":"math_evaluation/core/latex_normalize.py","file_url":"https://github.com/mario-math-reasoning/math_evaluation/blob/HEAD/math_evaluation/core/latex_normalize.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"08baa10ca8bb0932"}},{"code_sha256_prefix":"1c8ffc7a7070d5dd","entry":"further_string_normalize","repo":"mario-math-reasoning/math_evaluation","repo_kind":"official","path":"math_evaluation/core/latex_normalize.py","file_url":"https://github.com/mario-math-reasoning/math_evaluation/blob/HEAD/math_evaluation/core/latex_normalize.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1c8ffc7a7070d5dd"}},{"code_sha256_prefix":"045937dc669842fe","entry":"string_normalize","repo":"mario-math-reasoning/math_evaluation","repo_kind":"official","path":"math_evaluation/core/latex_normalize.py","file_url":"https://github.com/mario-math-reasoning/math_evaluation/blob/HEAD/math_evaluation/core/latex_normalize.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"045937dc669842fe"}},{"code_sha256_prefix":"a5269c2ba639e818","entry":"try_to_parse_function","repo":"mario-math-reasoning/math_evaluation","repo_kind":"official","path":"math_evaluation/core/type_evaluations.py","file_url":"https://github.com/mario-math-reasoning/math_evaluation/blob/HEAD/math_evaluation/core/type_evaluations.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a5269c2ba639e818"}},{"code_sha256_prefix":"067809ee50587572","entry":"is_equiv","repo":"mario-math-reasoning/mario_eval","repo_kind":"listed","path":"math_evaluation/core/evaluations.py","file_url":"https://github.com/mario-math-reasoning/mario_eval/blob/HEAD/math_evaluation/core/evaluations.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"067809ee50587572"}},{"code_sha256_prefix":"2d29fc30902b8474","entry":"is_equiv_complex","repo":"mario-math-reasoning/math_evaluation","repo_kind":"official","path":"math_evaluation/core/type_evaluations.py","file_url":"https://github.com/mario-math-reasoning/math_evaluation/blob/HEAD/math_evaluation/core/type_evaluations.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2d29fc30902b8474"}},{"code_sha256_prefix":"e5932e85cf891599","entry":"is_equiv_function","repo":"mario-math-reasoning/math_evaluation","repo_kind":"official","path":"math_evaluation/core/type_evaluations.py","file_url":"https://github.com/mario-math-reasoning/math_evaluation/blob/HEAD/math_evaluation/core/type_evaluations.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e5932e85cf891599"}},{"code_sha256_prefix":"41553ce1dca6b102","entry":"is_equiv_possible_intervals","repo":"mario-math-reasoning/mario_eval","repo_kind":"listed","path":"math_evaluation/core/evaluations.py","file_url":"https://github.com/mario-math-reasoning/mario_eval/blob/HEAD/math_evaluation/core/evaluations.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"41553ce1dca6b102"}},{"code_sha256_prefix":"db3f6dbcf3cdf416","entry":"is_latex_equiv","repo":"mario-math-reasoning/mario_eval","repo_kind":"listed","path":"math_evaluation/core/evaluations.py","file_url":"https://github.com/mario-math-reasoning/mario_eval/blob/HEAD/math_evaluation/core/evaluations.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"db3f6dbcf3cdf416"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}