{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/coverup-coverage-guided-llm-based-test","title":"CoverUp: Effective High Coverage Test Generation for Python","arxiv_id":"2403.16218","date":"2024-03-24","proceeding":null,"authors":["Juan Altmayer Pizzorno","Emery D. Berger"],"abstract":"Testing is an essential part of software development. Test generation tools attempt to automate the otherwise labor-intensive task of test creation, but generating high-coverage tests remains challenging. This paper proposes CoverUp, a novel approach to driving the generation of high-coverage Python regression tests. CoverUp combines coverage analysis, code context, and feedback in prompts that iteratively guide the LLM to generate tests that improve line and branch coverage. We evaluate our prototype CoverUp implementation across a benchmark of challenging code derived from open-source Python projects and show that CoverUp substantially improves on the state of the art. Compared to CodaMosa, a hybrid search/LLM-based test generator, CoverUp achieves a per-module median line+branch coverage of 80% (vs. 47%). Compared to MuTAP, a mutation- and LLM-based test generator, CoverUp achieves an overall line+branch coverage of 89% (vs. 77%). We also demonstrate that CoverUp's performance stems not only from the LLM used but from the combined effectiveness of its components.","url_abs":"https://arxiv.org/abs/2403.16218v4","url_pdf":"https://arxiv.org/pdf/2403.16218v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"coverup-coverage-guided-llm-based-test","repo_url":"https://github.com/plasma-umass/coverup","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"coverup-coverage-guided-llm-based-test","repo_url":"https://github.com/plasma-umass/coverup-eval","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"coverup-coverage-guided-llm-based-test","repo_url":"https://github.com/ZJU-ACES-ISE/ChatUniTest","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}}],"tasks":[{"task_slug":"software-testing","task_name":"software testing"}],"methods":[{"method_slug":"focus","method_name":"Focus"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2403.16218","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.16218"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/plasma-umass/coverup","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ZJU-ACES-ISE/ChatUniTest","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/plasma-umass/coverup-eval","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":12,"unverified":5},"by_repo_kind":{"official":{"samples":17,"ran":12,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"e780f2ac9b06a8c2","entry":"basedir","repo":"plasma-umass/coverup-eval","repo_kind":"official","path":"output/cm/ansible/coverup-tests/disabled_test_coverup_1137.py","file_url":"https://github.com/plasma-umass/coverup-eval/blob/HEAD/output/cm/ansible/coverup-tests/disabled_test_coverup_1137.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e780f2ac9b06a8c2"}},{"code_sha256_prefix":"145acc26153dc1f4","entry":"count_tokens","repo":"plasma-umass/coverup","repo_kind":"official","path":"src/coverup/llm.py","file_url":"https://github.com/plasma-umass/coverup/blob/HEAD/src/coverup/llm.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"145acc26153dc1f4"}},{"code_sha256_prefix":"ae89c975e4a8f393","entry":"custom_object_representer","repo":"plasma-umass/coverup-eval","repo_kind":"official","path":"output/cm/ansible/coverup-tests/disabled_test_coverup_1108.py","file_url":"https://github.com/plasma-umass/coverup-eval/blob/HEAD/output/cm/ansible/coverup-tests/disabled_test_coverup_1108.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ae89c975e4a8f393"}},{"code_sha256_prefix":"ecca514f32bdaa28","entry":"format_ranges","repo":"plasma-umass/coverup","repo_kind":"official","path":"src/coverup/utils.py","file_url":"https://github.com/plasma-umass/coverup/blob/HEAD/src/coverup/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ecca514f32bdaa28"}},{"code_sha256_prefix":"6f9ea5e7c5e3e194","entry":"get_global_imports","repo":"plasma-umass/coverup","repo_kind":"official","path":"src/coverup/codeinfo.py","file_url":"https://github.com/plasma-umass/coverup/blob/HEAD/src/coverup/codeinfo.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6f9ea5e7c5e3e194"}},{"code_sha256_prefix":"218a7174bbbafd7c","entry":"is_same_as_P","repo":"plasma-umass/coverup","repo_kind":"official","path":"src/coverup/logreader.py","file_url":"https://github.com/plasma-umass/coverup/blob/HEAD/src/coverup/logreader.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"218a7174bbbafd7c"}},{"code_sha256_prefix":"ec9b4903be33464d","entry":"lines_branches_do","repo":"plasma-umass/coverup","repo_kind":"official","path":"src/coverup/utils.py","file_url":"https://github.com/plasma-umass/coverup/blob/HEAD/src/coverup/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ec9b4903be33464d"}},{"code_sha256_prefix":"7bb4f87378c340a8","entry":"parse_file","repo":"plasma-umass/coverup","repo_kind":"official","path":"src/coverup/codeinfo.py","file_url":"https://github.com/plasma-umass/coverup/blob/HEAD/src/coverup/codeinfo.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7bb4f87378c340a8"}},{"code_sha256_prefix":"e134f05ecc5345c7","entry":"remove_omit","repo":"plasma-umass/coverup-eval","repo_kind":"official","path":"output/cm/ansible/coverup-tests/disabled_test_coverup_1123.py","file_url":"https://github.com/plasma-umass/coverup-eval/blob/HEAD/output/cm/ansible/coverup-tests/disabled_test_coverup_1123.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e134f05ecc5345c7"}},{"code_sha256_prefix":"a6bcfa534306ab26","entry":"subprocess_run","repo":"plasma-umass/coverup","repo_kind":"official","path":"src/coverup/utils.py","file_url":"https://github.com/plasma-umass/coverup/blob/HEAD/src/coverup/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a6bcfa534306ab26"}},{"code_sha256_prefix":"28aa58f0339c64af","entry":"test_file_path","repo":"plasma-umass/coverup","repo_kind":"official","path":"src/coverup/coverup.py","file_url":"https://github.com/plasma-umass/coverup/blob/HEAD/src/coverup/coverup.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"28aa58f0339c64af"}},{"code_sha256_prefix":"f840bd6712a9a03f","entry":"token_rate_limit_for_model","repo":"plasma-umass/coverup","repo_kind":"official","path":"src/coverup/llm.py","file_url":"https://github.com/plasma-umass/coverup/blob/HEAD/src/coverup/llm.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f840bd6712a9a03f"}},{"code_sha256_prefix":"def8599068e3dbf6","entry":"compute_cost","repo":"plasma-umass/coverup","repo_kind":"official","path":"src/coverup/llm.py","file_url":"https://github.com/plasma-umass/coverup/blob/HEAD/src/coverup/llm.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"def8599068e3dbf6"}},{"code_sha256_prefix":"9b6d481e57689ee5","entry":"get_info","repo":"plasma-umass/coverup","repo_kind":"official","path":"src/coverup/codeinfo.py","file_url":"https://github.com/plasma-umass/coverup/blob/HEAD/src/coverup/codeinfo.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9b6d481e57689ee5"}},{"code_sha256_prefix":"cc43dbef7115b69c","entry":"get_missing_coverage","repo":"plasma-umass/coverup","repo_kind":"official","path":"src/coverup/segment.py","file_url":"https://github.com/plasma-umass/coverup/blob/HEAD/src/coverup/segment.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cc43dbef7115b69c"}},{"code_sha256_prefix":"d3e7c47f194e80d7","entry":"new_test_file","repo":"plasma-umass/coverup","repo_kind":"official","path":"src/coverup/coverup.py","file_url":"https://github.com/plasma-umass/coverup/blob/HEAD/src/coverup/coverup.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d3e7c47f194e80d7"}},{"code_sha256_prefix":"f5192e3a7fcb2025","entry":"parse_log","repo":"plasma-umass/coverup","repo_kind":"official","path":"src/coverup/logreader.py","file_url":"https://github.com/plasma-umass/coverup/blob/HEAD/src/coverup/logreader.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f5192e3a7fcb2025"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}