{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/natural-adversarial-examples","title":"Natural Adversarial Examples","arxiv_id":"1907.07174","date":"2019-07-16","proceeding":"CVPR 2021 1","authors":["Dan Hendrycks","Kevin Zhao","Steven Basart","Jacob Steinhardt","Dawn Song"],"abstract":"We introduce two challenging datasets that reliably cause machine learning model performance to substantially degrade. The datasets are collected with a simple adversarial filtration technique to create datasets with limited spurious cues. Our datasets' real-world, unmodified examples transfer to various unseen models reliably, demonstrating that computer vision models have shared weaknesses. The first dataset is called ImageNet-A and is like the ImageNet test set, but it is far more challenging for existing models. We also curate an adversarial out-of-distribution detection dataset called ImageNet-O, which is the first out-of-distribution detection dataset created for ImageNet models. On ImageNet-A a DenseNet-121 obtains around 2% accuracy, an accuracy drop of approximately 90%, and its out-of-distribution detection performance on ImageNet-O is near random chance levels. We find that existing data augmentation techniques hardly boost performance, and using other public training datasets provides improvements that are limited. However, we find that improvements to computer vision architectures provide a promising path towards robust models.","url_abs":"https://arxiv.org/abs/1907.07174v4","url_pdf":"https://arxiv.org/pdf/1907.07174v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"natural-adversarial-examples","repo_url":"https://github.com/hendrycks/natural-adv-examples","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"natural-adversarial-examples","repo_url":"https://github.com/oskyhn/CNNs-Without-Borders","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"natural-adversarial-examples","repo_url":"https://github.com/zzzace2000/robust_cls_model","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"adversarial-attack","task_name":"Adversarial Attack"},{"task_slug":"data-augmentation","task_name":"Data Augmentation"},{"task_slug":"domain-generalization","task_name":"Domain Generalization"},{"task_slug":"out-of-distribution-detection","task_name":"Out-of-Distribution Detection"}],"methods":[{"method_slug":"1x1-convolution","method_name":"1x1 Convolution"},{"method_slug":"average-pooling","method_name":"Average Pooling"},{"method_slug":"batch-normalization","method_name":"Batch Normalization"},{"method_slug":"bottleneck-residual-block","method_name":"Bottleneck Residual Block"},{"method_slug":"convolution","method_name":"Convolution"},{"method_slug":"global-average-pooling","method_name":"Global Average Pooling"},{"method_slug":"kaiming-initialization","method_name":"Kaiming Initialization"},{"method_slug":"max-pooling","method_name":"Max Pooling"},{"method_slug":"relu","method_name":"ReLU"},{"method_slug":"residual-block","method_name":"Residual Block"},{"method_slug":"residual-connection","method_name":"Residual Connection"}],"datasets_introduced":[{"slug":"imagenet-a","name":"ImageNet-A","full_name":""},{"slug":"imagenet-o","name":"ImageNet-O","full_name":""}],"methods_introduced":[],"results":[{"leaderboard":"/sota/domain-generalization-on-imagenet-a","task":"Domain Generalization","dataset":"ImageNet-A","model":"ResNet-50","rank_in_archive_order":39,"of":39,"metrics":{"Top-1 accuracy %":"0"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1907.07174","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1907.07174"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/hendrycks/natural-adv-examples","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/oskyhn/CNNs-Without-Borders","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/zzzace2000/robust_cls_model","reach":null}],"summary":{"ran_fixture":1,"unverified":3},"by_repo_kind":{"official":{"samples":4,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"3e3cbd586608a39a","entry":"get_predictions","repo":"hendrycks/natural-adv-examples","repo_kind":"official","path":"eval_many_models.py","file_url":"https://github.com/hendrycks/natural-adv-examples/blob/HEAD/eval_many_models.py","link_basis":"plan_row","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3e3cbd586608a39a"}},{"code_sha256_prefix":"e0a2af598b5ea528","entry":"aurra","repo":"hendrycks/natural-adv-examples","repo_kind":"official","path":"calibration_tools.py","file_url":"https://github.com/hendrycks/natural-adv-examples/blob/HEAD/calibration_tools.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e0a2af598b5ea528"}},{"code_sha256_prefix":"509aa629e46cf950","entry":"calib_err","repo":"hendrycks/natural-adv-examples","repo_kind":"official","path":"calibration_tools.py","file_url":"https://github.com/hendrycks/natural-adv-examples/blob/HEAD/calibration_tools.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"509aa629e46cf950"}},{"code_sha256_prefix":"0a958e667c8c24c8","entry":"soft_f1","repo":"hendrycks/natural-adv-examples","repo_kind":"official","path":"calibration_tools.py","file_url":"https://github.com/hendrycks/natural-adv-examples/blob/HEAD/calibration_tools.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0a958e667c8c24c8"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}