{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/itbench-evaluating-ai-agents-across-diverse","title":"ITBench: Evaluating AI Agents across Diverse Real-World IT Automation Tasks","arxiv_id":"2502.05352","date":"2025-02-07","proceeding":null,"authors":["Saurabh Jha","Rohan Arora","Yuji Watanabe","Takumi Yanagawa","Yinfang Chen","Jackson Clark","Bhavya Bhavya","Mudit Verma","Harshit Kumar","Hirokuni Kitahara","Noah Zheutlin","Saki Takano","Divya Pathak","Felix George","Xinbo Wu","Bekir O. Turkkan","Gerard Vanloo","Michael Nidd","Ting Dai","Oishik Chatterjee","Pranjal Gupta","Suranjana Samanta","Pooja Aggarwal","Rong Lee","Pavankumar Murali","Jae-wook Ahn","Debanjana Kar","Ameet Rahane","Carlos Fonseca","Amit Paradkar","Yu Deng","Pratibha Moogi","Prateeti Mohapatra","Naoki Abe","Chandrasekhar Narayanaswami","Tianyin Xu","Lav R. Varshney","Ruchi Mahindru","Anca Sailer","Laura Shwartz","Daby Sow","Nicholas C. M. Fuller","Ruchir Puri"],"abstract":"Realizing the vision of using AI agents to automate critical IT tasks depends on the ability to measure and understand effectiveness of proposed solutions. We introduce ITBench, a framework that offers a systematic methodology for benchmarking AI agents to address real-world IT automation tasks. Our initial release targets three key areas: Site Reliability Engineering (SRE), Compliance and Security Operations (CISO), and Financial Operations (FinOps). The design enables AI researchers to understand the challenges and opportunities of AI agents for IT automation with push-button workflows and interpretable metrics. ITBench includes an initial set of 94 real-world scenarios, which can be easily extended by community contributions. Our results show that agents powered by state-of-the-art models resolve only 13.8% of SRE scenarios, 25.2% of CISO scenarios, and 0% of FinOps scenarios. We expect ITBench to be a key enabler of AI-driven IT automation that is correct, safe, and fast.","url_abs":"https://arxiv.org/abs/2502.05352v1","url_pdf":"https://arxiv.org/pdf/2502.05352v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"itbench-evaluating-ai-agents-across-diverse","repo_url":"https://github.com/IBM/itbench-sample-scenarios","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"itbench-evaluating-ai-agents-across-diverse","repo_url":"https://github.com/IBM/itbench-sre-agent","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"benchmarking","task_name":"Benchmarking"}],"methods":[{"method_slug":"set","method_name":"SET"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2502.05352","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2502.05352"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/IBM/itbench-sample-scenarios","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/IBM/itbench-sre-agent","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"unverified":6},"by_repo_kind":{"official":{"samples":6,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"2326ed24fc7d8d2e","entry":"check_violation_in_policy_reports","repo":"IBM/itbench-sample-scenarios","repo_kind":"official","path":"scenarios/ciso/4.upd-cis-b-k8s-kyverno/evaluation.py","file_url":"https://github.com/IBM/itbench-sample-scenarios/blob/HEAD/scenarios/ciso/4.upd-cis-b-k8s-kyverno/evaluation.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2326ed24fc7d8d2e"}},{"code_sha256_prefix":"3ed5ca00715a5d6b","entry":"get_leaderboard","repo":"IBM/itbench-sample-scenarios","repo_kind":"official","path":".github/workflows/leaderboard.py","file_url":"https://github.com/IBM/itbench-sample-scenarios/blob/HEAD/.github/workflows/leaderboard.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3ed5ca00715a5d6b"}},{"code_sha256_prefix":"e674c40f47fb9580","entry":"get_timestamp","repo":"IBM/itbench-sample-scenarios","repo_kind":"official","path":".github/workflows/leaderboard.py","file_url":"https://github.com/IBM/itbench-sample-scenarios/blob/HEAD/.github/workflows/leaderboard.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e674c40f47fb9580"}},{"code_sha256_prefix":"3190d9673df6bffb","entry":"load_yaml","repo":"IBM/itbench-sample-scenarios","repo_kind":"official","path":"scenarios/ciso/4.upd-cis-b-k8s-kyverno/evaluation.py","file_url":"https://github.com/IBM/itbench-sample-scenarios/blob/HEAD/scenarios/ciso/4.upd-cis-b-k8s-kyverno/evaluation.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3190d9673df6bffb"}},{"code_sha256_prefix":"0caea1961e8305b5","entry":"load_yaml_docs","repo":"IBM/itbench-sample-scenarios","repo_kind":"official","path":"scenarios/ciso/4.upd-cis-b-k8s-kyverno/evaluation.py","file_url":"https://github.com/IBM/itbench-sample-scenarios/blob/HEAD/scenarios/ciso/4.upd-cis-b-k8s-kyverno/evaluation.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0caea1961e8305b5"}},{"code_sha256_prefix":"2aa58855a8a2a319","entry":"parse_json_timedelta","repo":"IBM/itbench-sample-scenarios","repo_kind":"official","path":".github/workflows/leaderboard.py","file_url":"https://github.com/IBM/itbench-sample-scenarios/blob/HEAD/.github/workflows/leaderboard.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2aa58855a8a2a319"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}