{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/projecttest-a-project-level-llm-unit-test","title":"ProjectTest: A Project-level LLM Unit Test Generation Benchmark and Impact of Error Fixing Mechanisms","arxiv_id":"2502.06556","date":"2025-02-10","proceeding":null,"authors":["Yibo Wang","Congying Xia","Wenting Zhao","Jiangshu Du","Chunyu Miao","Zhongfen Deng","Philip S. Yu","Chen Xing"],"abstract":"Unit test generation has become a promising and important use case of LLMs. However, existing evaluation benchmarks for assessing LLM unit test generation capabilities focus on function- or class-level code rather than more practical and challenging project-level codebases. To address such limitation, we propose ProjectTest, a project-level benchmark for unit test generation covering Python, Java, and JavaScript. ProjectTest features 20 moderate-sized and high-quality projects per language. We evaluate nine frontier LLMs on ProjectTest and the results show that all frontier LLMs tested exhibit moderate performance on ProjectTest on Python and Java, highlighting the difficulty of ProjectTest. We also conduct a thorough error analysis, which shows that even frontier LLMs, such as Claude-3.5-Sonnet, have significant basic yet critical errors, including compilation and cascade errors. Motivated by this observation, we further evaluate all frontier LLMs under manual error-fixing and self-error-fixing scenarios to assess their potential when equipped with error-fixing mechanisms. Our code and dataset is available at \\href{https://github.com/YiboWANG214/ProjectTest}{ProjectTest}.","url_abs":"https://arxiv.org/abs/2502.06556v4","url_pdf":"https://arxiv.org/pdf/2502.06556v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"projecttest-a-project-level-llm-unit-test","repo_url":"https://github.com/yibowang214/projecttest","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[],"methods":[{"method_slug":"focus","method_name":"Focus"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2502.06556","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2502.06556"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/yibowang214/projecttest","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":15},"by_repo_kind":{"official":{"samples":15,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"02bacf43ead48d21","entry":"add_uppercase_char","repo":"yibowang214/projecttest","repo_kind":"official","path":"ProjectTest/Python/main/slugify.py","file_url":"https://github.com/yibowang214/projecttest/blob/HEAD/ProjectTest/Python/main/slugify.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"02bacf43ead48d21"}},{"code_sha256_prefix":"9954bd925b43cd43","entry":"cards2list","repo":"yibowang214/projecttest","repo_kind":"official","path":"ProjectTest/Python/main/mahjong.py","file_url":"https://github.com/yibowang214/projecttest/blob/HEAD/ProjectTest/Python/main/mahjong.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9954bd925b43cd43"}},{"code_sha256_prefix":"c091d9f8ecef34f9","entry":"compare_ranks","repo":"yibowang214/projecttest","repo_kind":"official","path":"ProjectTest/Python/main/limitholdem.py","file_url":"https://github.com/yibowang214/projecttest/blob/HEAD/ProjectTest/Python/main/limitholdem.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c091d9f8ecef34f9"}},{"code_sha256_prefix":"682e1d3f2db0d950","entry":"determine_winner","repo":"yibowang214/projecttest","repo_kind":"official","path":"ProjectTest/Python/main/limitholdem.py","file_url":"https://github.com/yibowang214/projecttest/blob/HEAD/ProjectTest/Python/main/limitholdem.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"682e1d3f2db0d950"}},{"code_sha256_prefix":"ac84ed81d3821073","entry":"determine_winner_straight","repo":"yibowang214/projecttest","repo_kind":"official","path":"ProjectTest/Python/main/limitholdem.py","file_url":"https://github.com/yibowang214/projecttest/blob/HEAD/ProjectTest/Python/main/limitholdem.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ac84ed81d3821073"}},{"code_sha256_prefix":"4e40be5721f8aa00","entry":"doudizhu_sort_card","repo":"yibowang214/projecttest","repo_kind":"official","path":"ProjectTest/Python/main/doudizhu.py","file_url":"https://github.com/yibowang214/projecttest/blob/HEAD/ProjectTest/Python/main/doudizhu.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4e40be5721f8aa00"}},{"code_sha256_prefix":"116d2d3794ede9c8","entry":"encode_cards","repo":"yibowang214/projecttest","repo_kind":"official","path":"ProjectTest/Python/main/mahjong.py","file_url":"https://github.com/yibowang214/projecttest/blob/HEAD/ProjectTest/Python/main/mahjong.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"116d2d3794ede9c8"}},{"code_sha256_prefix":"fbef5dbc6957b39b","entry":"get_landlord_score","repo":"yibowang214/projecttest","repo_kind":"official","path":"ProjectTest/Python/main/doudizhu.py","file_url":"https://github.com/yibowang214/projecttest/blob/HEAD/ProjectTest/Python/main/doudizhu.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fbef5dbc6957b39b"}},{"code_sha256_prefix":"649779b9072f895f","entry":"hashing_trick","repo":"yibowang214/projecttest","repo_kind":"official","path":"ProjectTest/Python/main/keras_preprocessing.py","file_url":"https://github.com/yibowang214/projecttest/blob/HEAD/ProjectTest/Python/main/keras_preprocessing.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"649779b9072f895f"}},{"code_sha256_prefix":"8f1600f856db0ca4","entry":"one_hot","repo":"yibowang214/projecttest","repo_kind":"official","path":"ProjectTest/Python/main/keras_preprocessing.py","file_url":"https://github.com/yibowang214/projecttest/blob/HEAD/ProjectTest/Python/main/keras_preprocessing.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8f1600f856db0ca4"}},{"code_sha256_prefix":"f6bb9657b399cfb4","entry":"pile2list","repo":"yibowang214/projecttest","repo_kind":"official","path":"ProjectTest/Python/main/mahjong.py","file_url":"https://github.com/yibowang214/projecttest/blob/HEAD/ProjectTest/Python/main/mahjong.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f6bb9657b399cfb4"}},{"code_sha256_prefix":"22582a48040295c6","entry":"rank2int","repo":"yibowang214/projecttest","repo_kind":"official","path":"ProjectTest/Python/main/leducholdem.py","file_url":"https://github.com/yibowang214/projecttest/blob/HEAD/ProjectTest/Python/main/leducholdem.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"22582a48040295c6"}},{"code_sha256_prefix":"5364b91b0022bbc9","entry":"slugify","repo":"yibowang214/projecttest","repo_kind":"official","path":"ProjectTest/Python/main/slugify.py","file_url":"https://github.com/yibowang214/projecttest/blob/HEAD/ProjectTest/Python/main/slugify.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5364b91b0022bbc9"}},{"code_sha256_prefix":"e0cbbf486684390f","entry":"smart_truncate","repo":"yibowang214/projecttest","repo_kind":"official","path":"ProjectTest/Python/main/slugify.py","file_url":"https://github.com/yibowang214/projecttest/blob/HEAD/ProjectTest/Python/main/slugify.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e0cbbf486684390f"}},{"code_sha256_prefix":"c2c19e1be4926d8d","entry":"text_to_word_sequence","repo":"yibowang214/projecttest","repo_kind":"official","path":"ProjectTest/Python/main/keras_preprocessing.py","file_url":"https://github.com/yibowang214/projecttest/blob/HEAD/ProjectTest/Python/main/keras_preprocessing.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c2c19e1be4926d8d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}