{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/code/call-api","entry":"call_api","source":"Syntology graph, per-sample; not an archive number","read_at":"2026-09-24T18:15:14+00:00","claim":"Names are grouped by exact entry-name string. Same-named routines are NOT asserted to be equivalent; 'ran' means executed on a synthesized fixture, not correctness. n_samples_ran = sum of by_status over every status except 'unverified' (ran_draft_wrong and ran_fixture are failures of Syntology's instrument, not of the code); n_papers_ran = papers with at least one such sample.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"n_papers":21,"n_papers_ran":4,"units":"n_samples, n_samples_ran, n_samples_fingerprinted and by_status count distinct code bodies (code_sha256); n_places and n_places_pointer_only count places, one per (paper, code body) pair, which is also the unit of the samples list","n_samples":23,"n_samples_ran":4,"n_samples_fingerprinted":0,"n_places":23,"n_places_pointer_only":9,"by_status":{"ran_honours":0,"ran_violates":0,"ran_draft_wrong":1,"ran_fixture":0,"ran":3,"unverified":19},"syntology":{"atlas_url":null,"mcp":null,"mcp_per_sample":{"tool":"get_code","arguments_in":"samples[].mcp_get_code"},"developers":"https://syntology.ai/developers"},"samples":[{"arxiv_id":"2606.12864","paper":"/paper/arxiv-2606-12864","title":"Beyond Problem Solving: UOJ-Bench for Evaluating Code Generation, Hacking, and Repair in Competitive Programming","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"hehezhou/UOJ-Bench","path":"utils/call_llm.py","file_url":"https://github.com/hehezhou/UOJ-Bench/blob/HEAD/utils/call_llm.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"8528f7a7b192f827","mcp_get_code":{"code_sha256":"8528f7a7b192f827"}},{"arxiv_id":"2605.28969","paper":"/paper/arxiv-2605-28969","title":"Beyond Recall: Behavioral Specification as an Interpretive Layer for AI Personalization","date":null,"month_inferred_from_arxiv_id":"2026-05","title_source":"syntology","repo":"agulaya24/beyond-recall","path":"baselayer/src/baselayer/api_client.py","file_url":"https://github.com/agulaya24/beyond-recall/blob/HEAD/baselayer/src/baselayer/api_client.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"06140119a1e1371b","mcp_get_code":{"code_sha256":"06140119a1e1371b"}},{"arxiv_id":"2605.07093","paper":"/paper/arxiv-2605-07093","title":"The Translation Tax Is Not a Scalar: A Counterfactual Audit of English-Source Cue Inheritance in Chinese Multilingual Benchmarks","date":null,"month_inferred_from_arxiv_id":"2026-05","title_source":"syntology","repo":"chi-mi-rvard/translation-tax-supplement","path":"run_annotation_pilot.py","file_url":"https://github.com/chi-mi-rvard/translation-tax-supplement/blob/HEAD/run_annotation_pilot.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"code_sha256_prefix":"7eb110e190bea6e2","mcp_get_code":{"code_sha256":"7eb110e190bea6e2"}},{"arxiv_id":"2603.16654","paper":"/paper/arxiv-2603-16654","title":"Omanic: Towards Step-wise Evaluation of Multi-hop Reasoning in Large Language Models","date":null,"month_inferred_from_arxiv_id":"2026-03","title_source":"syntology","repo":"XiaojieGu/Omanic","path":"eval/open_eval.py","file_url":"https://github.com/XiaojieGu/Omanic/blob/HEAD/eval/open_eval.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"daecce7b494c7457","mcp_get_code":{"code_sha256":"daecce7b494c7457"}},{"arxiv_id":"2601.11866","paper":"/paper/arxiv-2601-11866","title":"Advances in LLM Reasoning Enable Flexibility in Clinical Problem-Solving","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"bernardolab/mARC-Reasoning","path":"evaluate_from_api.py","file_url":"https://github.com/bernardolab/mARC-Reasoning/blob/HEAD/evaluate_from_api.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"d817db37ab3390f5","mcp_get_code":{"code_sha256":"d817db37ab3390f5"}},{"arxiv_id":"2601.10318","paper":"/paper/arxiv-2601-10318","title":"Boundary-Aware NL2SQL: Integrating Reliability through Hybrid Reward and Data Synthesis","date":null,"month_inferred_from_arxiv_id":"2026-01","title_source":"syntology","repo":"TianSongS/BAR-SQL","path":"model_test/trained_model_infer.py","file_url":"https://github.com/TianSongS/BAR-SQL/blob/HEAD/model_test/trained_model_infer.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"ef09c8678ddf2d2f","mcp_get_code":{"code_sha256":"ef09c8678ddf2d2f"}},{"arxiv_id":"2509.12158","paper":"/paper/arxiv-2509-12158","title":"Pun Unintended: LLMs and the Illusion of Humor Understanding","date":null,"month_inferred_from_arxiv_id":"2025-09","title_source":"syntology","repo":"alezanga/punintended","path":"utils/api.py","file_url":"https://github.com/alezanga/punintended/blob/HEAD/utils/api.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"4b837088d3c99a86","mcp_get_code":{"code_sha256":"4b837088d3c99a86"}},{"arxiv_id":"2504.04907","paper":null,"title":"arXiv:2504.04907","date":null,"month_inferred_from_arxiv_id":"2025-04","title_source":null,"repo":"Video-Bench/Video-Bench","path":"videobench/dynamicquality.py","file_url":"https://github.com/Video-Bench/Video-Bench/blob/HEAD/videobench/dynamicquality.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"4740acd70bea1016","mcp_get_code":{"code_sha256":"4740acd70bea1016"}},{"arxiv_id":"2504.01296","paper":"/paper/thinkprune-pruning-long-chain-of-thought-of","title":"ThinkPrune: Pruning Long Chain-of-Thought of LLMs via Reinforcement Learning","date":"2025-04-02","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"UCSB-NLP-Chang/ThinkPrune","path":"verl/evaluation/evalmath_main.py","file_url":"https://github.com/UCSB-NLP-Chang/ThinkPrune/blob/HEAD/verl/evaluation/evalmath_main.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"aaa4bac6dc85512a","mcp_get_code":{"code_sha256":"aaa4bac6dc85512a"}},{"arxiv_id":"2411.03284","paper":"/paper/smoa-improving-multi-agent-large-language","title":"SMoA: Improving Multi-agent Large Language Models with Sparse Mixture-of-Agents","date":"2024-11-05","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"david-li0406/smoa","path":"eval_ceb.py","file_url":"https://github.com/david-li0406/smoa/blob/HEAD/eval_ceb.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"c4be2ee7c659a23f","mcp_get_code":{"code_sha256":"c4be2ee7c659a23f"}},{"arxiv_id":"2410.05262","paper":"/paper/turtlebench-evaluating-top-language-models","title":"TurtleBench: Evaluating Top Language Models via Real-World Yes/No Puzzles","date":"2024-10-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"mazzzystar/TurtleBench","path":"models.py","file_url":"https://github.com/mazzzystar/TurtleBench/blob/HEAD/models.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"b6a420ef1adb8bef","mcp_get_code":{"code_sha256":"b6a420ef1adb8bef"}},{"arxiv_id":"2410.02694","paper":"/paper/helmet-how-to-evaluate-long-context-language","title":"HELMET: How to Evaluate Long-Context Language Models Effectively and Thoroughly","date":"2024-10-03","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"princeton-nlp/helmet","path":"model_utils.py","file_url":"https://github.com/princeton-nlp/helmet/blob/HEAD/model_utils.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"38db855956f81312","mcp_get_code":{"code_sha256":"38db855956f81312"}},{"arxiv_id":"2410.00812","paper":"/paper/a-generative-framework-to-bridge-data-driven","title":"Generative causal testing to bridge data-driven models and scientific theories in language neuroscience","date":"2024-10-01","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"microsoft/automated-explanations","path":"experiments/05_run_ngrams_gpt.py","file_url":"https://github.com/microsoft/automated-explanations/blob/HEAD/experiments/05_run_ngrams_gpt.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"91e9d8ef7d0f2766","mcp_get_code":{"code_sha256":"91e9d8ef7d0f2766"}},{"arxiv_id":"2409.02257","paper":"/paper/mmlu-pro-evaluating-higher-order-reasoning","title":"MMLU-Pro+: Evaluating Higher-Order Reasoning and Shortcut Learning in LLMs","date":"2024-09-03","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"asgsaeid/mmlu-pro-plus","path":"evaluate_from_api.py","file_url":"https://github.com/asgsaeid/mmlu-pro-plus/blob/HEAD/evaluate_from_api.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e5a8f0fb32623146","mcp_get_code":{"code_sha256":"e5a8f0fb32623146"}},{"arxiv_id":"2409.02253","paper":"/paper/how-to-determine-the-preferred-image","title":"How to Determine the Preferred Image Distribution of a Black-Box Vision-Language Model?","date":"2024-09-03","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"asgsaeid/cad_vqa","path":"cad_qa_eval_api.py","file_url":"https://github.com/asgsaeid/cad_vqa/blob/HEAD/cad_qa_eval_api.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"5f0ad3a3aab7add5","mcp_get_code":{"code_sha256":"5f0ad3a3aab7add5"}},{"arxiv_id":"2310.08129","paper":"/paper/tailored-visions-enhancing-text-to-image","title":"Tailored Visions: Enhancing Text-to-Image Generation with Personalized Prompt Rewriting","date":"2023-10-12","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"zzjchen/tailored-visions","path":"apiuse.py","file_url":"https://github.com/zzjchen/tailored-visions/blob/HEAD/apiuse.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"238b375eb1515ee4","mcp_get_code":{"code_sha256":"238b375eb1515ee4"}},{"arxiv_id":"2306.03030","paper":"/paper/benchmarking-large-language-models-on-cmexam","title":"Benchmarking Large Language Models on CMExam -- A Comprehensive Chinese Medical Exam Dataset","date":"2023-06-05","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"williamliujl/cmexam","path":"src/evaluation/evaluate_gpt_result.py","file_url":"https://github.com/williamliujl/cmexam/blob/HEAD/src/evaluation/evaluate_gpt_result.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"7959c2beea6950cc","mcp_get_code":{"code_sha256":"7959c2beea6950cc"}},{"arxiv_id":"Huang_OPERA_Alleviating_Hallucination_in_Multi-Modal_Large_Language_Models_via_Over-Trust_CVPR_2024_paper","paper":null,"title":"arXiv:Huang_OPERA_Alleviating_Hallucination_in_Multi-Modal_Large_Language_Models_via_Over-Trust_CVPR_2024_paper","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"shikiw/OPERA","path":"gpt4v_eval.py","file_url":"https://github.com/shikiw/OPERA/blob/HEAD/gpt4v_eval.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"8e4ee2c3bf99a3cf","mcp_get_code":{"code_sha256":"8e4ee2c3bf99a3cf"}},{"arxiv_id":"2025.findings-emnlp.549","paper":null,"title":"arXiv:2025.findings-emnlp.549","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"ZackZikaiXiao/LongWeave","path":"core/serve/dlc.py","file_url":"https://github.com/ZackZikaiXiao/LongWeave/blob/HEAD/core/serve/dlc.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"8842c94c17b3a581","mcp_get_code":{"code_sha256":"8842c94c17b3a581"}},{"arxiv_id":"2025.findings-emnlp.549","paper":null,"title":"arXiv:2025.findings-emnlp.549","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"ZackZikaiXiao/LongWeave","path":"core/serve/dlc_r.py","file_url":"https://github.com/ZackZikaiXiao/LongWeave/blob/HEAD/core/serve/dlc_r.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"dae5ea6e381242b6","mcp_get_code":{"code_sha256":"dae5ea6e381242b6"}},{"arxiv_id":"2025.findings-emnlp.549","paper":null,"title":"arXiv:2025.findings-emnlp.549","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"ZackZikaiXiao/LongWeave","path":"core/serve/huggingface.py","file_url":"https://github.com/ZackZikaiXiao/LongWeave/blob/HEAD/core/serve/huggingface.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"293eca342addab1d","mcp_get_code":{"code_sha256":"293eca342addab1d"}},{"arxiv_id":"2025.emnlp-main.640","paper":null,"title":"arXiv:2025.emnlp-main.640","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"aszala/DiagrammerGPT","path":"src/openai_handler.py","file_url":"https://github.com/aszala/DiagrammerGPT/blob/HEAD/src/openai_handler.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"717c5b7dedd08579","mcp_get_code":{"code_sha256":"717c5b7dedd08579"}},{"arxiv_id":"2024.findings-acl.848","paper":null,"title":"arXiv:2024.findings-acl.848","date":null,"month_inferred_from_arxiv_id":null,"title_source":null,"repo":"CAS-SIAT-ConsistencyAI/NUMCoT","path":"src/code/experience_first/common.py","file_url":"https://github.com/CAS-SIAT-ConsistencyAI/NUMCoT/blob/HEAD/src/code/experience_first/common.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"CC0-1.0","inline_ok":true,"code_sha256_prefix":"98bce14cf93ff0d1","mcp_get_code":{"code_sha256":"98bce14cf93ff0d1"}}]}