{"url":"/task/humaneval","name":"HumanEval","slug":"humaneval","description_markdown":null,"categories":[],"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","slug_source":"archive_url"},"counts":{"papers_tagged":264,"papers_with_code":135,"benchmarks":0,"benchmark_tables_in_archive":0,"benchmark_tables_shown":0,"benchmark_tables_withheld_as_spam":0,"benchmark_definition":"a leaderboard table with at least one row; benchmark_tables_shown also counts the zero-row tables; benchmark_tables_in_archive adds the tables withheld as spam","datasets":1,"subtasks":0,"parent_tasks":0},"benchmarks":[],"datasets":[{"url":"/dataset/humaneval","name":"HumanEval","full_name":"","num_papers_in_archive":1201}],"subtasks":[],"parent_tasks":[],"papers":{"order":"repositories listed in the archive (desc), then date (desc); the archive holds no stars","population":"papers tagged with this task that list at least one repository in the archive","shown":30,"of":135,"tagged_in_all":264,"items":[{"url":"/paper/evaluating-large-language-models-trained-on","title":"Evaluating Large Language Models Trained on Code","date":"2021-07-07","arxiv_id":"2107.03374","repositories_listed":13,"syntology":{"n":39,"n_ran":6,"n_unverified":33,"n_pointer_only":2}},{"url":"/paper/a-conversational-paradigm-for-program","title":"CodeGen: An Open Large Language Model for Code with Multi-Turn Program Synthesis","date":"2022-03-25","arxiv_id":"2203.13474","repositories_listed":8,"syntology":{"n":10,"n_ran":10,"n_unverified":0,"n_pointer_only":0}},{"url":"/paper/chatglm-a-family-of-large-language-models","title":"ChatGLM: A Family of Large Language Models from GLM-130B to GLM-4 All Tools","date":"2024-06-18","arxiv_id":"2406.12793","repositories_listed":7,"syntology":{"n":29,"n_ran":15,"n_unverified":14,"n_pointer_only":0}},{"url":"/paper/qwen2-technical-report","title":"Qwen2 Technical Report","date":"2024-07-15","arxiv_id":"2407.10671","repositories_listed":6,"syntology":null},{"url":"/paper/reflexion-language-agents-with-verbal","title":"Reflexion: Language Agents with Verbal Reinforcement Learning","date":"2023-03-20","arxiv_id":"2303.11366","repositories_listed":5,"syntology":{"n":9,"n_ran":2,"n_unverified":7,"n_pointer_only":0}},{"url":"/paper/wizardcoder-empowering-code-large-language","title":"WizardCoder: Empowering Code Large Language Models with Evol-Instruct","date":"2023-06-14","arxiv_id":"2306.08568","repositories_listed":4,"syntology":{"n":7,"n_ran":3,"n_unverified":4,"n_pointer_only":1}},{"url":"/paper/starcoder-may-the-source-be-with-you","title":"StarCoder: may the source be with you!","date":"2023-05-09","arxiv_id":"2305.06161","repositories_listed":4,"syntology":{"n":2,"n_ran":2,"n_unverified":0,"n_pointer_only":0}},{"url":"/paper/soap-enhancing-efficiency-of-generated-code","title":"EffiLearner: Enhancing Efficiency of Generated Code via Self-Optimization","date":"2024-05-24","arxiv_id":"2405.15189","repositories_listed":3,"syntology":{"n":17,"n_ran":12,"n_unverified":5,"n_pointer_only":0}},{"url":"/paper/rlhf-workflow-from-reward-modeling-to-online","title":"RLHF Workflow: From Reward Modeling to Online RLHF","date":"2024-05-13","arxiv_id":"2405.07863","repositories_listed":3,"syntology":{"n":1,"n_ran":0,"n_unverified":1,"n_pointer_only":1}},{"url":"/paper/magicoder-source-code-is-all-you-need","title":"Magicoder: Empowering Code Generation with OSS-Instruct","date":"2023-12-04","arxiv_id":"2312.02120","repositories_listed":3,"syntology":{"n":2,"n_ran":1,"n_unverified":1,"n_pointer_only":0}},{"url":"/paper/octopack-instruction-tuning-code-large","title":"OctoPack: Instruction Tuning Code Large Language Models","date":"2023-08-14","arxiv_id":"2308.07124","repositories_listed":3,"syntology":{"n":13,"n_ran":9,"n_unverified":4,"n_pointer_only":0}},{"url":"/paper/recode-robustness-evaluation-of-code","title":"ReCode: Robustness Evaluation of Code Generation Models","date":"2022-12-20","arxiv_id":"2212.10264","repositories_listed":3,"syntology":{"n":5,"n_ran":5,"n_unverified":0,"n_pointer_only":0}},{"url":"/paper/selfcodealign-self-alignment-for-code","title":"SelfCodeAlign: Self-Alignment for Code Generation","date":"2024-10-31","arxiv_id":"2410.24198","repositories_listed":2,"syntology":{"n":37,"n_ran":13,"n_unverified":24,"n_pointer_only":0}},{"url":"/paper/training-language-models-to-self-correct-via","title":"Training Language Models to Self-Correct via Reinforcement Learning","date":"2024-09-19","arxiv_id":"2409.12917","repositories_listed":2,"syntology":null},{"url":"/paper/planning-in-natural-language-improves-llm","title":"Planning In Natural Language Improves LLM Search For Code Generation","date":"2024-09-05","arxiv_id":"2409.03733","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_unverified":0,"n_pointer_only":1}},{"url":"/paper/a-survey-on-large-language-models-for-code","title":"A Survey on Large Language Models for Code Generation","date":"2024-06-01","arxiv_id":"2406.00515","repositories_listed":2,"syntology":{"n":1,"n_ran":0,"n_unverified":1,"n_pointer_only":0}},{"url":"/paper/mapcoder-multi-agent-code-generation-for","title":"MapCoder: Multi-Agent Code Generation for Competitive Problem Solving","date":"2024-05-18","arxiv_id":"2405.11403","repositories_listed":2,"syntology":{"n":10,"n_ran":0,"n_unverified":10,"n_pointer_only":0}},{"url":"/paper/codeultrafeedback-an-llm-as-a-judge-dataset","title":"CodeUltraFeedback: An LLM-as-a-Judge Dataset for Aligning Large Language Models to Coding Preferences","date":"2024-03-14","arxiv_id":"2403.09032","repositories_listed":2,"syntology":{"n":1,"n_ran":1,"n_unverified":0,"n_pointer_only":0}},{"url":"/paper/inficoder-eval-systematically-evaluating-the","title":"InfiBench: Evaluating the Question-Answering Capabilities of Code Large Language Models","date":"2024-03-11","arxiv_id":"2404.07940","repositories_listed":2,"syntology":{"n":13,"n_ran":11,"n_unverified":2,"n_pointer_only":0}},{"url":"/paper/generation-meets-verification-accelerating","title":"Generation Meets Verification: Accelerating Large Language Model Inference with Smart Parallel Auto-Correct Decoding","date":"2024-02-19","arxiv_id":"2402.11809","repositories_listed":2,"syntology":{"n":11,"n_ran":8,"n_unverified":3,"n_pointer_only":0}},{"url":"/paper/unsupervised-evaluation-of-code-llms-with","title":"Unsupervised Evaluation of Code LLMs with Round-Trip Correctness","date":"2024-02-13","arxiv_id":"2402.08699","repositories_listed":2,"syntology":{"n":4,"n_ran":2,"n_unverified":2,"n_pointer_only":0}},{"url":"/paper/language-agent-tree-search-unifies-reasoning","title":"Language Agent Tree Search Unifies Reasoning Acting and Planning in Language Models","date":"2023-10-06","arxiv_id":"2310.04406","repositories_listed":2,"syntology":{"n":9,"n_ran":7,"n_unverified":2,"n_pointer_only":0}},{"url":"/paper/baichuan-2-open-large-scale-language-models","title":"Baichuan 2: Open Large-scale Language Models","date":"2023-09-19","arxiv_id":"2309.10305","repositories_listed":2,"syntology":null},{"url":"/paper/code-llama-open-foundation-models-for-code","title":"Code Llama: Open Foundation Models for Code","date":"2023-08-24","arxiv_id":"2308.12950","repositories_listed":2,"syntology":{"n":5,"n_ran":3,"n_unverified":2,"n_pointer_only":4}},{"url":"/paper/classeval-a-manually-crafted-benchmark-for","title":"ClassEval: A Manually-Crafted Benchmark for Evaluating LLMs on Class-level Code Generation","date":"2023-08-03","arxiv_id":"2308.01861","repositories_listed":2,"syntology":null},{"url":"/paper/codet5-open-code-large-language-models-for","title":"CodeT5+: Open Code Large Language Models for Code Understanding and Generation","date":"2023-05-13","arxiv_id":"2305.07922","repositories_listed":2,"syntology":{"n":4,"n_ran":3,"n_unverified":1,"n_pointer_only":0}},{"url":"/paper/codegeex-a-pre-trained-model-for-code","title":"CodeGeeX: A Pre-Trained Model for Code Generation with Multilingual Benchmarking on HumanEval-X","date":"2023-03-30","arxiv_id":"2303.17568","repositories_listed":2,"syntology":{"n":8,"n_ran":2,"n_unverified":6,"n_pointer_only":0}},{"url":"/paper/multi-lingual-evaluation-of-code-generation","title":"Multi-lingual Evaluation of Code Generation Models","date":"2022-10-26","arxiv_id":"2210.14868","repositories_listed":2,"syntology":{"n":6,"n_ran":1,"n_unverified":5,"n_pointer_only":0}},{"url":"/paper/rethinking-verification-for-llm-code","title":"Rethinking Verification for LLM Code Generation: From Generation to Testing","date":"2025-07-09","arxiv_id":"2507.06920","repositories_listed":1,"syntology":null},{"url":"/paper/any4-learned-4-bit-numeric-representation-for","title":"any4: Learned 4-bit Numeric Representation for LLMs","date":"2025-07-07","arxiv_id":"2507.04610","repositories_listed":1,"syntology":{"n":2,"n_ran":2,"n_unverified":0,"n_pointer_only":2}}],"syntology_records":25,"syntology_note":"a paper without a record is not a recorded non-run: it may lack an arXiv id or simply be absent from the graph layer"},"description_links":{"kept":0,"unwrapped_to_text":0,"bare_urls_linked":0,"relative_images_dropped":0,"rule":"internal links are kept only when the target slug exists in the catalog"},"syntology":{"read_at":"2026-09-24T18:15:14+00:00","claim":"Per-sample execution status on synthesized fixtures ('ran N of M samples'); not a correctness claim and not a ranking signal.","status_vocabulary":{"ran_honours":"ran, honoured the contract we drafted","ran_violates":"ran, violated the contract we drafted","ran_draft_wrong":"ran; our contract draft was wrong, not the code","ran_fixture":"ran; our fixture could not drive it","ran":"ran on a synthesized input","unverified":"unverified (harvested, no recorded run)"}},"not_shown":{"libraries":"the archive has no per-task library table","trend_sparklines":"the Trend column of the benchmarks table was a rendered image; it is not in the archive","social_and_latest_sorts":"stars and social signals are not in the archive"}}