{"url":"/dataset/sudo-dataset","name":"SUDO Dataset","full_name":null,"description_markdown":"SUDO is a benchmark of 50 real-world malicious tasks designed to evaluate LLM-based computer agents in live desktop and web environments. It covers critical risk domains—including system security, content safety, societal harms, and privacy violations—based on the AirBench taxonomy. The dataset supports fine-grained evaluation using task-specific checklists and can be used to assess model misuse potential, build safer agents, or guide alignment research.","description_withheld":null,"homepage":"","introduced_date":"2025-03-26","introduced_date_note":null,"introduced_by":{"paper":"/paper/sudo-rm-rf-agentic-security","title":"sudo rm -rf agentic_security","first_author":"Sejin Lee","url":null},"license":null,"modalities":[{"name":"Texts","url":"/datasets/modality/texts"}],"tasks":[{"name":"Red Teaming","url":"/task/red-teaming","datasets_with_task":"/datasets/task/red-teaming"},{"name":"Real-World Adversarial Attack","url":"/task/real-world-adversarial-attack","datasets_with_task":"/datasets/task/real-world-adversarial-attack"},{"name":"AI and Safety","url":"/task/ai-and-safety","datasets_with_task":"/datasets/task/ai-and-safety"},{"name":"Safety Alignment","url":"/task/safety-alignment","datasets_with_task":"/datasets/task/safety-alignment"},{"name":"LLM Jailbreak","url":"/task/llm-jailbreak","datasets_with_task":"/datasets/task/llm-jailbreak"}],"languages":[{"name":"English","url":"/datasets/language/english"}],"variants":["SUDO Dataset"],"data_loaders":[],"num_papers_in_archive":1,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28"},"benchmarks":[{"leaderboard":"/sota/red-teaming-on-sudo-dataset","task":"Red Teaming","dataset_variant":"SUDO Dataset","rows":1,"metrics":["Attack Success Rate"],"first_row_in_archive_order":{"model":"SUDO","paper":"/paper/sudo-rm-rf-agentic-security","metrics":{"Attack Success Rate":"41%"},"code_links":[{"title":"AIM-Intelligence/SUDO","url":"https://github.com/AIM-Intelligence/SUDO"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"}],"papers_with_a_benchmark_row":[{"paper":"/paper/sudo-rm-rf-agentic-security","title":"sudo rm -rf agentic_security","date":"2025-03-26","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":2,"samples_ran":2,"samples_unverified":0,"pointer_only_for_licence":2,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}}],"syntology_totals":{"read_at":"2026-09-24T18:15:14+00:00","papers_with_samples":1,"samples_harvested":2,"samples_ran":2,"samples_unverified":0,"pointer_only_for_licence":2,"papers_with_no_sample_that_ran":0,"note":"the per-paper counts above, summed; not a rate"},"papers_note":"The archive never published its papers-using-dataset list; these are papers with a leaderboard row on this dataset's benchmarks."}