{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/code/estimate-pass-at-k","entry":"estimate_pass_at_k","source":"Syntology graph, per-sample; not an archive number","read_at":"2026-09-24T18:15:14+00:00","claim":"Names are grouped by exact entry-name string. Same-named routines are NOT asserted to be equivalent; 'ran' means executed on a synthesized fixture, not correctness. n_samples_ran = sum of by_status over every status except 'unverified' (ran_draft_wrong and ran_fixture are failures of Syntology's instrument, not of the code); n_papers_ran = papers with at least one such sample.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"},"n_papers":28,"n_papers_ran":20,"units":"n_samples, n_samples_ran, n_samples_fingerprinted and by_status count distinct code bodies (code_sha256); n_places and n_places_pointer_only count places, one per (paper, code body) pair, which is also the unit of the samples list","n_samples":14,"n_samples_ran":8,"n_samples_fingerprinted":4,"n_places":29,"n_places_pointer_only":6,"by_status":{"ran_honours":0,"ran_violates":0,"ran_draft_wrong":2,"ran_fixture":0,"ran":6,"unverified":6},"syntology":{"atlas_url":null,"mcp":null,"mcp_per_sample":{"tool":"get_code","arguments_in":"samples[].mcp_get_code"},"developers":"https://syntology.ai/developers"},"samples":[{"arxiv_id":"2607.24884","paper":"/paper/arxiv-2607-24884","title":"Beyond \"What to Retrieve\": Uncertainty in Retrieval-Augmented Code Generation","date":null,"month_inferred_from_arxiv_id":"2026-07","title_source":"syntology","repo":"Rocky5502/OpenCoder_V1","path":"opencoder/evaluation/metrics.py","file_url":"https://github.com/Rocky5502/OpenCoder_V1/blob/HEAD/opencoder/evaluation/metrics.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"42c81d3f43739308","mcp_get_code":{"code_sha256":"42c81d3f43739308"}},{"arxiv_id":"2607.19395","paper":"/paper/arxiv-2607-19395","title":"From Trajectories to Prefixes: Reusing Teacher Trajectories via Replayed Prefixes and Online Continuation","date":null,"month_inferred_from_arxiv_id":"2026-07","title_source":"syntology","repo":"HappynessI/Prefix_GRPO","path":"legacy_eval/eval/eval_webshop_vllm_server.py","file_url":"https://github.com/HappynessI/Prefix_GRPO/blob/HEAD/legacy_eval/eval/eval_webshop_vllm_server.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"bfc71d0afa64f3f5","mcp_get_code":{"code_sha256":"bfc71d0afa64f3f5"}},{"arxiv_id":"2606.05859","paper":"/paper/arxiv-2606-05859","title":"TARPO: Token-Wise Latent-Explicit Reasoning via Action-Routing Policy Optimization","date":null,"month_inferred_from_arxiv_id":"2026-06","title_source":"syntology","repo":"NKU-LITI/TARPO-master","path":"humanevaleval.py","file_url":"https://github.com/NKU-LITI/TARPO-master/blob/HEAD/humanevaleval.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e080db2eaa81fdb3","mcp_get_code":{"code_sha256":"e080db2eaa81fdb3"}},{"arxiv_id":"2604.04356","paper":"/paper/arxiv-2604-04356","title":"REAM: Merging Improves Pruning of Experts in LLMs","date":null,"month_inferred_from_arxiv_id":"2026-04","title_source":"syntology","repo":"zai-org/glm-simple-evals","path":"evals/livecodebench_eval.py","file_url":"https://github.com/zai-org/glm-simple-evals/blob/HEAD/evals/livecodebench_eval.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"9df10eb02582b09f","mcp_get_code":{"code_sha256":"9df10eb02582b09f"}},{"arxiv_id":"2602.13773","paper":"/paper/arxiv-2602-13773","title":"On Representation Redundancy in Large-Scale Instruction Tuning Data Selection","date":null,"month_inferred_from_arxiv_id":"2026-02","title_source":"syntology","repo":"tdano1/CRDS","path":"main/eval/codex_humaneval/evaluation.py","file_url":"https://github.com/tdano1/CRDS/blob/HEAD/main/eval/codex_humaneval/evaluation.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"ff29b26b4a0b7f51","mcp_get_code":{"code_sha256":"ff29b26b4a0b7f51"}},{"arxiv_id":"2602.11639","paper":"/paper/arxiv-2602-11639","title":"PACE: Prefix-Protected and Difficulty-Aware Compression for Efficient Reasoning","date":null,"month_inferred_from_arxiv_id":"2026-02","title_source":"syntology","repo":"LiveCodeBench/LiveCodeBench","path":"lcb_runner/evaluation/pass_k_utils.py","file_url":"https://github.com/LiveCodeBench/LiveCodeBench/blob/HEAD/lcb_runner/evaluation/pass_k_utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"9df10eb02582b09f","mcp_get_code":{"code_sha256":"9df10eb02582b09f"}},{"arxiv_id":"2602.02979","paper":"/paper/arxiv-2602-02979","title":"CPMöbius: Iterative Coach-Player Reasoning for Data-Free Reinforcement Learning","date":null,"month_inferred_from_arxiv_id":"2026-02","title_source":"syntology","repo":"thunlp/CPMobius","path":"evaluation/utils/evaluation.py","file_url":"https://github.com/thunlp/CPMobius/blob/HEAD/evaluation/utils/evaluation.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"e080db2eaa81fdb3","mcp_get_code":{"code_sha256":"e080db2eaa81fdb3"}},{"arxiv_id":"2502.01456","paper":"/paper/process-reinforcement-through-implicit","title":"Process Reinforcement through Implicit Rewards","date":"2025-02-03","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"prime-rl/prime","path":"eval/utils/evaluation.py","file_url":"https://github.com/prime-rl/prime/blob/HEAD/eval/utils/evaluation.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"e080db2eaa81fdb3","mcp_get_code":{"code_sha256":"e080db2eaa81fdb3"}},{"arxiv_id":"2411.17501","paper":"/paper/inference-scaling-scriptsize-mathtt-f-laws","title":"Inference Scaling fLaws: The Limits of LLM Resampling with Imperfect Verifiers","date":"2024-11-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"benediktstroebl/inference-scaling-limits","path":"race/codeeval/evaluation_utils_evalplus.py","file_url":"https://github.com/benediktstroebl/inference-scaling-limits/blob/HEAD/race/codeeval/evaluation_utils_evalplus.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"5a283b50e4b14e37","mcp_get_code":{"code_sha256":"5a283b50e4b14e37"}},{"arxiv_id":"2407.11470","paper":"/paper/beyond-correctness-benchmarking-multi","title":"Beyond Correctness: Benchmarking Multi-dimensional Code Generation for Large Language Models","date":"2024-07-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"jszheng21/race","path":"race/codeeval/evaluation_utils_evalplus.py","file_url":"https://github.com/jszheng21/race/blob/HEAD/race/codeeval/evaluation_utils_evalplus.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"5a283b50e4b14e37","mcp_get_code":{"code_sha256":"5a283b50e4b14e37"}},{"arxiv_id":"2407.01910","paper":"/paper/mg-verilog-multi-grained-dataset-towards","title":"MG-Verilog: Multi-grained Dataset Towards Enhanced LLM-assisted Verilog Generation","date":"2024-07-02","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"luke-avionics/mg-verilog","path":"verilog_eval/verilog_eval/evaluation.py","file_url":"https://github.com/luke-avionics/mg-verilog/blob/HEAD/verilog_eval/verilog_eval/evaluation.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"0fe5b6fed06ae660","mcp_get_code":{"code_sha256":"0fe5b6fed06ae660"}},{"arxiv_id":"2406.18532","paper":"/paper/symbolic-learning-enables-self-evolving","title":"Symbolic Learning Enables Self-Evolving Agents","date":"2024-06-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"aiwaves-cn/agents","path":"src/agents/datasets/humaneval.py","file_url":"https://github.com/aiwaves-cn/agents/blob/HEAD/src/agents/datasets/humaneval.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"9c2911e17309fc8f","mcp_get_code":{"code_sha256":"9c2911e17309fc8f"}},{"arxiv_id":"2405.05347","paper":"/paper/benchmarking-educational-program-repair","title":"Benchmarking Educational Program Repair","date":"2024-05-08","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"koutchemecharles/gaied_nips23","path":"src/utils/evaluation.py","file_url":"https://github.com/koutchemecharles/gaied_nips23/blob/HEAD/src/utils/evaluation.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"df629018634c1265","mcp_get_code":{"code_sha256":"df629018634c1265"}},{"arxiv_id":"2404.02078","paper":"/paper/advancing-llm-reasoning-generalists-with","title":"Advancing LLM Reasoning Generalists with Preference Trees","date":"2024-04-02","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"openbmb/eurus","path":"eval/utils/evaluation.py","file_url":"https://github.com/openbmb/eurus/blob/HEAD/eval/utils/evaluation.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"e080db2eaa81fdb3","mcp_get_code":{"code_sha256":"e080db2eaa81fdb3"}},{"arxiv_id":"2402.14852","paper":"/paper/humaneval-on-latest-gpt-models-2024","title":"HumanEval on Latest GPT Models -- 2024","date":"2024-02-20","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"daniel442li/gpt-human-eval","path":"human_eval/evaluation.py","file_url":"https://github.com/daniel442li/gpt-human-eval/blob/HEAD/human_eval/evaluation.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e080db2eaa81fdb3","mcp_get_code":{"code_sha256":"e080db2eaa81fdb3"}},{"arxiv_id":"2401.16405","paper":"/paper/scaling-sparse-fine-tuning-to-large-language","title":"Scaling Sparse Fine-Tuning to Large Language Models","date":"2024-01-29","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ducdauge/sft-llm","path":"eval/codex_humaneval/evaluation.py","file_url":"https://github.com/ducdauge/sft-llm/blob/HEAD/eval/codex_humaneval/evaluation.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e080db2eaa81fdb3","mcp_get_code":{"code_sha256":"e080db2eaa81fdb3"}},{"arxiv_id":"2401.06628","paper":"/paper/oop-object-oriented-programming-evaluation","title":"OOP: Object-Oriented Programming Evaluation Benchmark for Large Language Models","date":"2024-01-12","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"alphadl/oop-eval","path":"oop_evaluate/evaluation.py","file_url":"https://github.com/alphadl/oop-eval/blob/HEAD/oop_evaluate/evaluation.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e080db2eaa81fdb3","mcp_get_code":{"code_sha256":"e080db2eaa81fdb3"}},{"arxiv_id":"2310.20329","paper":"/paper/instructcoder-empowering-language-models-for","title":"InstructCoder: Instruction Tuning Large Language Models for Code Editing","date":"2023-10-31","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"qishenghu/CodeInstruct","path":"edit_eval/evaluation.py","file_url":"https://github.com/qishenghu/CodeInstruct/blob/HEAD/edit_eval/evaluation.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e080db2eaa81fdb3","mcp_get_code":{"code_sha256":"e080db2eaa81fdb3"}},{"arxiv_id":"2310.06830","paper":"/paper/lemur-harmonizing-natural-language-and-code","title":"Lemur: Harmonizing Natural Language and Code for Language Agents","date":"2023-10-10","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"OpenLemur/Lemur","path":"xchat/eval/mbpp/evaluation.py","file_url":"https://github.com/OpenLemur/Lemur/blob/HEAD/xchat/eval/mbpp/evaluation.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"af79b5e511f124e0","mcp_get_code":{"code_sha256":"af79b5e511f124e0"}},{"arxiv_id":"2309.13345","paper":"/paper/bamboo-a-comprehensive-benchmark-for","title":"BAMBOO: A Comprehensive Benchmark for Evaluating Long Text Modeling Capacities of Large Language Models","date":"2023-09-23","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"rucaibox/bamboo","path":"private_eval/evaluation.py","file_url":"https://github.com/rucaibox/bamboo/blob/HEAD/private_eval/evaluation.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"code_sha256_prefix":"e080db2eaa81fdb3","mcp_get_code":{"code_sha256":"e080db2eaa81fdb3"}},{"arxiv_id":"2309.00071","paper":"/paper/yarn-efficient-context-window-extension-of","title":"YaRN: Efficient Context Window Extension of Large Language Models","date":"2023-08-31","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"qwenlm/qwq","path":"eval/eval/livecodebench_v5_utils/pass_k_utils.py","file_url":"https://github.com/qwenlm/qwq/blob/HEAD/eval/eval/livecodebench_v5_utils/pass_k_utils.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"9df10eb02582b09f","mcp_get_code":{"code_sha256":"9df10eb02582b09f"}},{"arxiv_id":"2307.02477","paper":"/paper/reasoning-or-reciting-exploring-the","title":"Reasoning or Reciting? Exploring the Capabilities and Limitations of Language Models Through Counterfactual Tasks","date":"2023-07-05","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"zhaofengwu/counterfactual-evaluation","path":"programming/generation/human_eval/evaluation.py","file_url":"https://github.com/zhaofengwu/counterfactual-evaluation/blob/HEAD/programming/generation/human_eval/evaluation.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"e080db2eaa81fdb3","mcp_get_code":{"code_sha256":"e080db2eaa81fdb3"}},{"arxiv_id":"2306.03438","paper":"/paper/large-language-models-of-code-fail-at","title":"Large Language Models of Code Fail at Completing Code with Potential Bugs","date":"2023-06-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"amazon-science/buggy-code-completion","path":"src/metrics/evaluation.py","file_url":"https://github.com/amazon-science/buggy-code-completion/blob/HEAD/src/metrics/evaluation.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"af8375df02e6b247","mcp_get_code":{"code_sha256":"af8375df02e6b247"}},{"arxiv_id":"2303.03004","paper":"/paper/xcodeeval-a-large-scale-multilingual","title":"xCodeEval: A Large Scale Multilingual Multitask Benchmark for Code Understanding, Generation, Translation and Retrieval","date":"2023-03-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ntunlp/execeval","path":"eval_scripts/eval_passk.py","file_url":"https://github.com/ntunlp/execeval/blob/HEAD/eval_scripts/eval_passk.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"1e081510f01d4d42","mcp_get_code":{"code_sha256":"1e081510f01d4d42"}},{"arxiv_id":"2303.03004","paper":"/paper/xcodeeval-a-large-scale-multilingual","title":"xCodeEval: A Large Scale Multilingual Multitask Benchmark for Code Understanding, Generation, Translation and Retrieval","date":"2023-03-06","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"ntunlp/xCodeEval","path":"evaluation/apr/get_result.py","file_url":"https://github.com/ntunlp/xCodeEval/blob/HEAD/evaluation/apr/get_result.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"8c30a56df3c52cef","mcp_get_code":{"code_sha256":"8c30a56df3c52cef"}},{"arxiv_id":"2302.08468","paper":"/paper/lever-learning-to-verify-language-to-code","title":"LEVER: Learning to Verify Language-to-Code Generation with Execution","date":"2023-02-16","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"niansong1996/lever","path":"few_shot/codex.py","file_url":"https://github.com/niansong1996/lever/blob/HEAD/few_shot/codex.py","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"3a39b10c8a429dc8","mcp_get_code":{"code_sha256":"3a39b10c8a429dc8"}},{"arxiv_id":"2210.14868","paper":"/paper/multi-lingual-evaluation-of-code-generation","title":"Multi-lingual Evaluation of Code Generation Models","date":"2022-10-26","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"amazon-research/mbxp-exec-eval","path":"mxeval/evaluation.py","file_url":"https://github.com/amazon-research/mbxp-exec-eval/blob/HEAD/mxeval/evaluation.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"code_sha256_prefix":"af79b5e511f124e0","mcp_get_code":{"code_sha256":"af79b5e511f124e0"}},{"arxiv_id":"2207.14255","paper":"/paper/efficient-training-of-language-models-to-fill","title":"Efficient Training of Language Models to Fill in the Middle","date":"2022-07-28","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"openai/human-eval-infilling","path":"human_eval_infilling/evaluation.py","file_url":"https://github.com/openai/human-eval-infilling/blob/HEAD/human_eval_infilling/evaluation.py","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"code_sha256_prefix":"af79b5e511f124e0","mcp_get_code":{"code_sha256":"af79b5e511f124e0"}},{"arxiv_id":"2107.03374","paper":"/paper/evaluating-large-language-models-trained-on","title":"Evaluating Large Language Models Trained on Code","date":"2021-07-07","month_inferred_from_arxiv_id":null,"title_source":"archive","repo":"2796gaurav/human-eval","path":"human_eval/evaluation.py","file_url":"https://github.com/2796gaurav/human-eval/blob/HEAD/human_eval/evaluation.py","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":false,"code_sha256_prefix":"e080db2eaa81fdb3","mcp_get_code":{"code_sha256":"e080db2eaa81fdb3"}}]}