{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/cweval-outcome-driven-evaluation-on","title":"CWEval: Outcome-driven Evaluation on Functionality and Security of LLM Code Generation","arxiv_id":"2501.08200","date":"2025-01-14","proceeding":null,"authors":["Jinjun Peng","Leyi Cui","Kele Huang","Junfeng Yang","Baishakhi Ray"],"abstract":"Large Language Models (LLMs) have significantly aided developers by generating or assisting in code writing, enhancing productivity across various tasks. While identifying incorrect code is often straightforward, detecting vulnerabilities in functionally correct code is more challenging, especially for developers with limited security knowledge, which poses considerable security risks of using LLM-generated code and underscores the need for robust evaluation benchmarks that assess both functional correctness and security. Current benchmarks like CyberSecEval and SecurityEval attempt to solve it but are hindered by unclear and impractical specifications, failing to assess both functionality and security accurately. To tackle these deficiencies, we introduce CWEval, a novel outcome-driven evaluation framework designed to enhance the evaluation of secure code generation by LLMs. This framework not only assesses code functionality but also its security simultaneously with high-quality task specifications and outcome-driven test oracles which provides high accuracy. Coupled with CWEval-bench, a multilingual, security-critical coding benchmark, CWEval provides a rigorous empirical security evaluation on LLM-generated code, overcoming previous benchmarks' shortcomings. Through our evaluations, CWEval reveals a notable portion of functional but insecure code produced by LLMs, and shows a serious inaccuracy of previous evaluations, ultimately contributing significantly to the field of secure code generation. We open-source our artifact at: https://github.com/Co1lin/CWEval .","url_abs":"https://arxiv.org/abs/2501.08200v1","url_pdf":"https://arxiv.org/pdf/2501.08200v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"cweval-outcome-driven-evaluation-on","repo_url":"https://github.com/co1lin/cweval","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"code-generation","task_name":"Code Generation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2501.08200","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2501.08200"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/co1lin/cweval","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"unverified":9},"by_repo_kind":{"official":{"samples":9,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"f17a43f998cfc7df","entry":"complete_code","repo":"co1lin/cweval","repo_kind":"official","path":"cweval/commons.py","file_url":"https://github.com/co1lin/cweval/blob/HEAD/cweval/commons.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f17a43f998cfc7df"}},{"code_sha256_prefix":"71c899bff40e2f27","entry":"evaluate_expression","repo":"co1lin/cweval","repo_kind":"official","path":"benchmark/core/py/cwe_095_0_task.py","file_url":"https://github.com/co1lin/cweval/blob/HEAD/benchmark/core/py/cwe_095_0_task.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"71c899bff40e2f27"}},{"code_sha256_prefix":"a5c728cb2b373aa8","entry":"extract_tar_to_path","repo":"co1lin/cweval","repo_kind":"official","path":"benchmark/core/py/cwe_022_2_task.py","file_url":"https://github.com/co1lin/cweval/blob/HEAD/benchmark/core/py/cwe_022_2_task.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a5c728cb2b373aa8"}},{"code_sha256_prefix":"4d6d25f26c33184f","entry":"get_code_from","repo":"co1lin/cweval","repo_kind":"official","path":"cweval/commons.py","file_url":"https://github.com/co1lin/cweval/blob/HEAD/cweval/commons.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4d6d25f26c33184f"}},{"code_sha256_prefix":"94f25c979d82e9ef","entry":"get_ls_result","repo":"co1lin/cweval","repo_kind":"official","path":"benchmark/core/py/cwe_078_0_task.py","file_url":"https://github.com/co1lin/cweval/blob/HEAD/benchmark/core/py/cwe_078_0_task.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"94f25c979d82e9ef"}},{"code_sha256_prefix":"7ccb900b8c3c6a35","entry":"get_target_for_redirect","repo":"co1lin/cweval","repo_kind":"official","path":"benchmark/core/py/cwe_020_0_task.py","file_url":"https://github.com/co1lin/cweval/blob/HEAD/benchmark/core/py/cwe_020_0_task.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7ccb900b8c3c6a35"}},{"code_sha256_prefix":"19e488691d74ec82","entry":"get_welcome_message","repo":"co1lin/cweval","repo_kind":"official","path":"benchmark/core/py/cwe_079_0_task.py","file_url":"https://github.com/co1lin/cweval/blob/HEAD/benchmark/core/py/cwe_079_0_task.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"19e488691d74ec82"}},{"code_sha256_prefix":"089dd199b73ad689","entry":"read_file_from_dir","repo":"co1lin/cweval","repo_kind":"official","path":"benchmark/core/py/cwe_022_0_task.py","file_url":"https://github.com/co1lin/cweval/blob/HEAD/benchmark/core/py/cwe_022_0_task.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"089dd199b73ad689"}},{"code_sha256_prefix":"1d423fe6d1bcc35a","entry":"run_in_subprocess","repo":"co1lin/cweval","repo_kind":"official","path":"cweval/commons.py","file_url":"https://github.com/co1lin/cweval/blob/HEAD/cweval/commons.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1d423fe6d1bcc35a"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}