{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2601-20886","title":"IDE-Bench: Evaluating Large Language Models as IDE Agents on Real-World Software Engineering Tasks","arxiv_id":"2601.20886","date":"2026-01-28","proceeding":null,"authors":["Spencer Mateega","Jeff Yang","Tiana Costello","Shaurya Jadhav","Nicole Tian","Agustin Garcinuño"],"abstract":"IDE-Bench is a comprehensive framework for evaluating AI IDE agents on real-world software engineering tasks through an IDE-native tool interface. We present a Dockerized test harness that goes beyond raw terminal execution, granting models a structured tool ecosystem that represents AI-native IDEs like Cursor and Windsurf. By providing high-level abstractions for codebase search, structured file editing, and tools for testing full-stack applications, IDE-Bench evaluates an agent's ability to act as a true engineering collaborator. For evaluation and to prevent training data contamination, we created 80 tasks across eight never-published repositories spanning C/C++, Java, and MERN stacks, representing modern tech stack production scenarios, including feature implementation, bug fixing, refactoring, and performance optimization that mirror daily developer workflows in private codebases. Our benchmark is the first to systematically correlate agent-reported intent with successful project-level modifications in a multi-language, full-stack environment on completely uncontaminated code. We release IDE-Bench and a public leaderboard at: https://ide-bench.com.","url_abs":"https://arxiv.org/abs/2601.20886","url_pdf":"https://arxiv.org/pdf/2601.20886","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2601.20886","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2601.20886"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/AfterQuery/IDE-Bench","reach":null}],"summary":{"unverified":11},"by_repo_kind":{"found_in_text":{"samples":11,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":11,"samples":[{"code_sha256_prefix":"903ff41546e0a2b2","entry":"analyze_task_requirements","repo":"AfterQuery/IDE-Bench","repo_kind":"found_in_text","path":"agent_utils.py","file_url":"https://github.com/AfterQuery/IDE-Bench/blob/HEAD/agent_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"903ff41546e0a2b2"}},{"code_sha256_prefix":"0b0336a1e98bf884","entry":"create_dataset_init_container_spec","repo":"AfterQuery/IDE-Bench","repo_kind":"found_in_text","path":"k8s/dataset-manager.py","file_url":"https://github.com/AfterQuery/IDE-Bench/blob/HEAD/k8s/dataset-manager.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0b0336a1e98bf884"}},{"code_sha256_prefix":"82fc1a223cadd478","entry":"extract_lab_quality_metrics","repo":"AfterQuery/IDE-Bench","repo_kind":"found_in_text","path":"grader.py","file_url":"https://github.com/AfterQuery/IDE-Bench/blob/HEAD/grader.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"82fc1a223cadd478"}},{"code_sha256_prefix":"5624281672064fc0","entry":"generate_implementation_guidance","repo":"AfterQuery/IDE-Bench","repo_kind":"found_in_text","path":"agent_utils.py","file_url":"https://github.com/AfterQuery/IDE-Bench/blob/HEAD/agent_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5624281672064fc0"}},{"code_sha256_prefix":"d08c77feec2db0d0","entry":"get_task_names","repo":"AfterQuery/IDE-Bench","repo_kind":"found_in_text","path":"utilities/run_all_tasks.py","file_url":"https://github.com/AfterQuery/IDE-Bench/blob/HEAD/utilities/run_all_tasks.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d08c77feec2db0d0"}},{"code_sha256_prefix":"76a50ae044b5280e","entry":"is_test_related_operation","repo":"AfterQuery/IDE-Bench","repo_kind":"found_in_text","path":"harness.py","file_url":"https://github.com/AfterQuery/IDE-Bench/blob/HEAD/harness.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"76a50ae044b5280e"}},{"code_sha256_prefix":"643ddd2087bdb835","entry":"parse_task_description","repo":"AfterQuery/IDE-Bench","repo_kind":"found_in_text","path":"util.py","file_url":"https://github.com/AfterQuery/IDE-Bench/blob/HEAD/util.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"643ddd2087bdb835"}},{"code_sha256_prefix":"dd39d5d1d310a4ee","entry":"run_command_in_container","repo":"AfterQuery/IDE-Bench","repo_kind":"found_in_text","path":"docker_utils.py","file_url":"https://github.com/AfterQuery/IDE-Bench/blob/HEAD/docker_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"dd39d5d1d310a4ee"}},{"code_sha256_prefix":"fdc169c7f72f0efe","entry":"run_task","repo":"AfterQuery/IDE-Bench","repo_kind":"found_in_text","path":"utilities/run_all_tasks.py","file_url":"https://github.com/AfterQuery/IDE-Bench/blob/HEAD/utilities/run_all_tasks.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fdc169c7f72f0efe"}},{"code_sha256_prefix":"ae23672cfe688ee2","entry":"sanitize_traceback","repo":"AfterQuery/IDE-Bench","repo_kind":"found_in_text","path":"harness.py","file_url":"https://github.com/AfterQuery/IDE-Bench/blob/HEAD/harness.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ae23672cfe688ee2"}},{"code_sha256_prefix":"7f8ad5e71d5a9ddd","entry":"security_wrapper","repo":"AfterQuery/IDE-Bench","repo_kind":"found_in_text","path":"harness.py","file_url":"https://github.com/AfterQuery/IDE-Bench/blob/HEAD/harness.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7f8ad5e71d5a9ddd"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.LG","source":"arxiv_2026.jsonl"},"syntology_extracted_results":null}