{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/code/is-equiv","entry":"is_equiv","source":"Syntology graph, per-sample; not an archive number","read_at":"2026-09-24T18:15:14+00:00","claim":"Names are grouped by exact entry-name string. Same-named routines are NOT asserted to be equivalent; 'ran' means executed on a synthesized fixture, not correctness. n_samples_ran = sum of by_status over every status except 'unverified' (ran_draft_wrong and ran_fixture are failures of Syntology's instrument, not of the code); n_papers_ran = papers with at least one such sample.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"n_papers":22,"n_papers_ran":18,"units":"n_samples, n_samples_ran, n_samples_fingerprinted and by_status count distinct code bodies (code_sha256); n_places and n_places_pointer_only count places, one per (paper, code body) pair, which is also the unit of the samples list","n_samples":9,"n_samples_ran":5,"n_samples_fingerprinted":0,"n_places":22,"n_places_pointer_only":7,"by_status":{"ran_honours":0,"ran_violates":2,"ran_draft_wrong":0,"ran_fixture":0,"ran":3,"unverified":4},"syntology":{"atlas_url":null,"mcp":null,"mcp_per_sample":{"tool":"get_code","arguments_in":"samples[].mcp_get_code"},"developers":"https://syntology.ai/developers"},"samples":[{"arxiv_id":"2609.00787","paper":"/paper/arxiv-2609-00787","title":"StudyBench: Can Self-Evolution Squeeze Textbooks for Olympiad Capability?","date":null,"month_inferred_from_arxiv_id":"2026-09","title_source":"syntology","repo":"thunlp/StudyBench","path":"eval/math_equivalence.py","file_url":"https://github.com/thunlp/StudyBench/blob/HEAD/eval/math_equivalence.py","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"20063c675d638a77","mcp_get_code":{"code_sha256":"20063c675d638a77"}},{"arxiv_id":"2606.18195","paper":"/paper/arxiv-2606-18195","title":"Learning from the Self-future: On-policy Self-distillation for dLLMs","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"xingzhejun/d-OPSD","path":"d-opsd/math500_utils.py","file_url":"https://github.com/xingzhejun/d-OPSD/blob/HEAD/d-opsd/math500_utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e88118a107b326d1","mcp_get_code":{"code_sha256":"e88118a107b326d1"}},{"arxiv_id":"2605.29398","paper":"/paper/arxiv-2605-29398","title":"GDSD: Reinforcement Learning as Guided Denoiser Self-Distillation for Diffusion Language Models","date":null,"month_inferred_from_arxiv_id":"2026-05","title_source":"syntology","repo":"GaryBall/GDSD","path":"gdsd/utils/math500_utils.py","file_url":"https://github.com/GaryBall/GDSD/blob/HEAD/gdsd/utils/math500_utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"b2845803d938cb26","mcp_get_code":{"code_sha256":"b2845803d938cb26"}},{"arxiv_id":"2604.23626","paper":"/paper/arxiv-2604-23626","title":"GraphPlanner: Graph Memory-Augmented Agentic Routing for Multi-Agent LLMs","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"ulab-uiuc/GraphPlanner","path":"router_planner/shared/math_eval.py","file_url":"https://github.com/ulab-uiuc/GraphPlanner/blob/HEAD/router_planner/shared/math_eval.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"34ed87891e855695","mcp_get_code":{"code_sha256":"34ed87891e855695"}},{"arxiv_id":"2604.13552","paper":"/paper/arxiv-2604-13552","title":"Training-Free Test-Time Contrastive Learning for Large Language Models","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"KevinSCUTer/TF-TTCL","path":"core/source/compare/selection_kernel.py","file_url":"https://github.com/KevinSCUTer/TF-TTCL/blob/HEAD/core/source/compare/selection_kernel.py","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"5015a8c78052f4f0","mcp_get_code":{"code_sha256":"5015a8c78052f4f0"}},{"arxiv_id":"2602.06462","paper":"/paper/arxiv-2602-06462","title":"Diffusion-State Policy Optimization for Masked Diffusion Language Models","date":null,"month_inferred_from_arxiv_id":"2026-02","title_source":"syntology","repo":"dllm-reasoning/d1","path":"diffu-grpo/math500_utils.py","file_url":"https://github.com/dllm-reasoning/d1/blob/HEAD/diffu-grpo/math500_utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"e88118a107b326d1","mcp_get_code":{"code_sha256":"e88118a107b326d1"}},{"arxiv_id":"2511.13223","paper":"/paper/arxiv-2511-13223","title":"TokenSqueeze: Performance-Preserving Compression for Reasoning LLMs","date":null,"month_inferred_from_arxiv_id":"2025-11","title_source":"syntology","repo":"zhangyx1122/TokenSqueeze","path":"utils/math500_verify.py","file_url":"https://github.com/zhangyx1122/TokenSqueeze/blob/HEAD/utils/math500_verify.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e88118a107b326d1","mcp_get_code":{"code_sha256":"e88118a107b326d1"}},{"arxiv_id":"2505.14625","paper":"/paper/tinyv-reducing-false-negatives-in","title":"TinyV: Reducing False Negatives in Verification Improves RL for LLM Reasoning","date":"2025-05-20","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"uw-nsl/tinyv","path":"analysis_tool/verl_reward_score/math.py","file_url":"https://github.com/uw-nsl/tinyv/blob/HEAD/analysis_tool/verl_reward_score/math.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e88118a107b326d1","mcp_get_code":{"code_sha256":"e88118a107b326d1"}},{"arxiv_id":"2505.13308","paper":"/paper/seek-in-the-dark-reasoning-via-test-time","title":"Seek in the Dark: Reasoning via Test-Time Instance-Level Policy Gradient in Latent Space","date":"2025-05-19","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"bigai-nlco/latentseek","path":"src/extract_judge_answer/math_equivalent_MATH.py","file_url":"https://github.com/bigai-nlco/latentseek/blob/HEAD/src/extract_judge_answer/math_equivalent_MATH.py","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"20063c675d638a77","mcp_get_code":{"code_sha256":"20063c675d638a77"}},{"arxiv_id":"2504.20157","paper":"/paper/toward-evaluative-thinking-meta-policy","title":"Toward Evaluative Thinking: Meta Policy Optimization with Evolving Reward Models","date":"2025-04-28","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"minnesotanlp/mpo","path":"trl/extras/mpo/rm_math_reasoning.py","file_url":"https://github.com/minnesotanlp/mpo/blob/HEAD/trl/extras/mpo/rm_math_reasoning.py","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"code_sha256_prefix":"20063c675d638a77","mcp_get_code":{"code_sha256":"20063c675d638a77"}},{"arxiv_id":"2503.02324","paper":"/paper/promptcot-synthesizing-olympiad-level","title":"PromptCoT: Synthesizing Olympiad-level Problems for Mathematical Reasoning in Large Language Models","date":"2025-03-04","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"zhaoxlpku/promptcot","path":"eval/math_equivalence.py","file_url":"https://github.com/zhaoxlpku/promptcot/blob/HEAD/eval/math_equivalence.py","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"20063c675d638a77","mcp_get_code":{"code_sha256":"20063c675d638a77"}},{"arxiv_id":"2502.11133","paper":"/paper/masrouter-learning-to-route-llms-for-multi","title":"MasRouter: Learning to Route LLMs for Multi-Agent Systems","date":"2025-02-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"yanweiyue/masrouter","path":"Datasets/math_dataset.py","file_url":"https://github.com/yanweiyue/masrouter/blob/HEAD/Datasets/math_dataset.py","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"20063c675d638a77","mcp_get_code":{"code_sha256":"20063c675d638a77"}},{"arxiv_id":"2411.07681","paper":"/paper/what-do-learning-dynamics-reveal-about","title":"What Do Learning Dynamics Reveal About Generalization in LLM Reasoning?","date":"2024-11-12","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"katiekang1998/reasoning_generalization","path":"utils.py","file_url":"https://github.com/katiekang1998/reasoning_generalization/blob/HEAD/utils.py","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"20063c675d638a77","mcp_get_code":{"code_sha256":"20063c675d638a77"}},{"arxiv_id":"2411.04282","paper":"/paper/language-models-are-hidden-reasoners","title":"Language Models are Hidden Reasoners: Unlocking Latent Reasoning Capabilities via Self-Rewarding","date":"2024-11-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"salesforceairesearch/latro","path":"LaTRO/utils/eval_utils.py","file_url":"https://github.com/salesforceairesearch/latro/blob/HEAD/LaTRO/utils/eval_utils.py","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"20063c675d638a77","mcp_get_code":{"code_sha256":"20063c675d638a77"}},{"arxiv_id":"2404.13925","paper":"/paper/mario-eval-evaluate-your-math-llm-with-your","title":"MARIO Eval: Evaluate Your Math LLM with your Math LLM--A mathematical dataset evaluation toolkit","date":"2024-04-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mario-math-reasoning/mario_eval","path":"math_evaluation/core/evaluations.py","file_url":"https://github.com/mario-math-reasoning/mario_eval/blob/HEAD/math_evaluation/core/evaluations.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"067809ee50587572","mcp_get_code":{"code_sha256":"067809ee50587572"}},{"arxiv_id":"2404.02078","paper":"/paper/advancing-llm-reasoning-generalists-with","title":"Advancing LLM Reasoning Generalists with Preference Trees","date":"2024-04-02","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"openbmb/eurus","path":"eval/utils/math_equivalence.py","file_url":"https://github.com/openbmb/eurus/blob/HEAD/eval/utils/math_equivalence.py","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"20063c675d638a77","mcp_get_code":{"code_sha256":"20063c675d638a77"}},{"arxiv_id":"2402.05120","paper":"/paper/more-agents-is-all-you-need","title":"More Agents Is All You Need","date":"2024-02-03","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"moreagentsisallyouneed/agentforest","path":"src/math_equivalence.py","file_url":"https://github.com/moreagentsisallyouneed/agentforest/blob/HEAD/src/math_equivalence.py","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"20063c675d638a77","mcp_get_code":{"code_sha256":"20063c675d638a77"}},{"arxiv_id":"2310.09168","paper":"/paper/explore-instruct-enhancing-domain-specific","title":"Explore-Instruct: Enhancing Domain-Specific Instruction Coverage through Active Exploration","date":"2023-10-13","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"fanqiwan/Explore-Instruct","path":"eval/auto_eval/math/math_equivalence.py","file_url":"https://github.com/fanqiwan/Explore-Instruct/blob/HEAD/eval/auto_eval/math/math_equivalence.py","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"20063c675d638a77","mcp_get_code":{"code_sha256":"20063c675d638a77"}},{"arxiv_id":"2310.05914","paper":"/paper/neftune-noisy-embeddings-improve-instruction","title":"NEFTune: Noisy Embeddings Improve Instruction Finetuning","date":"2023-10-09","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"akjindal53244/arithmo","path":"eval/MATH/MATH_compute_metric_zero_shot_CoT.py","file_url":"https://github.com/akjindal53244/arithmo/blob/HEAD/eval/MATH/MATH_compute_metric_zero_shot_CoT.py","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"20063c675d638a77","mcp_get_code":{"code_sha256":"20063c675d638a77"}},{"arxiv_id":"2303.12712","paper":"/paper/sparks-of-artificial-general-intelligence","title":"Sparks of Artificial General Intelligence: Early experiments with GPT-4","date":"2023-03-22","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"emrgnt-cmplxty/zero-shot-replication","path":"zero_shot_replication/core/math_helpers.py","file_url":"https://github.com/emrgnt-cmplxty/zero-shot-replication/blob/HEAD/zero_shot_replication/core/math_helpers.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"19df601915034cc3","mcp_get_code":{"code_sha256":"19df601915034cc3"}},{"arxiv_id":"openreview_QrC8OgQyOI","paper":null,"title":"arXiv:openreview_QrC8OgQyOI","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"viiika/Prism","path":"Dream/Dream_Prism/metrics/math500_eval.py","file_url":"https://github.com/viiika/Prism/blob/HEAD/Dream/Dream_Prism/metrics/math500_eval.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"eaf84570592463bd","mcp_get_code":{"code_sha256":"eaf84570592463bd"}},{"arxiv_id":"2024.findings-emnlp.407","paper":null,"title":"arXiv:2024.findings-emnlp.407","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"amayuelas/multi-agent-attack","path":"multiagent_debate/math_equivalence.py","file_url":"https://github.com/amayuelas/multi-agent-attack/blob/HEAD/multiagent_debate/math_equivalence.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"4fa5d4203318fbad","mcp_get_code":{"code_sha256":"4fa5d4203318fbad"}}]}