{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/mnist-c-a-robustness-benchmark-for-computer","title":"MNIST-C: A Robustness Benchmark for Computer Vision","arxiv_id":"1906.02337","date":"2019-06-05","proceeding":null,"authors":["Norman Mu","Justin Gilmer"],"abstract":"We introduce the MNIST-C dataset, a comprehensive suite of 15 corruptions applied to the MNIST test set, for benchmarking out-of-distribution robustness in computer vision. Through several experiments and visualizations we demonstrate that our corruptions significantly degrade performance of state-of-the-art computer vision models while preserving the semantic content of the test images. In contrast to the popular notion of adversarial robustness, our model-agnostic corruptions do not seek worst-case performance but are instead designed to be broad and diverse, capturing multiple failure modes of modern models. In fact, we find that several previously published adversarial defenses significantly degrade robustness as measured by MNIST-C. We hope that our benchmark serves as a useful tool for future work in designing systems that are able to learn robust feature representations that capture the underlying semantics of the input.","url_abs":"https://arxiv.org/abs/1906.02337v1","url_pdf":"https://arxiv.org/pdf/1906.02337v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"mnist-c-a-robustness-benchmark-for-computer","repo_url":"https://github.com/google-research/mnist-c","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":null},{"paper_slug":"mnist-c-a-robustness-benchmark-for-computer","repo_url":"https://github.com/testingautomated-usi/fashion-mnist-c","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"adversarial-robustness","task_name":"Adversarial Robustness"},{"task_slug":"benchmarking","task_name":"Benchmarking"}],"methods":[],"datasets_introduced":[{"slug":"mnist-c","name":"MNIST-C","full_name":""}],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/1906.02337","atlas_url":"https://app.syntology.ai/?focus=1906.02337","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1906.02337"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/google-research/mnist-c","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/testingautomated-usi/fashion-mnist-c","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":1,"ran":4,"unverified":3},"by_repo_kind":{"official":{"samples":1,"ran":1,"repositories":1},"listed":{"samples":6,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":1,"samples":[{"code_sha256_prefix":"f210d4ac7d4969d7","entry":"clipped_zoom","repo":"google-research/mnist-c","repo_kind":"official","path":"corruptions.py","file_url":"https://github.com/google-research/mnist-c/blob/HEAD/corruptions.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f210d4ac7d4969d7"}},{"code_sha256_prefix":"07a9b58ec8d910eb","entry":"disk","repo":"testingautomated-usi/fashion-mnist-c","repo_kind":"listed","path":"mnist_c.py","file_url":"https://github.com/testingautomated-usi/fashion-mnist-c/blob/HEAD/mnist_c.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"07a9b58ec8d910eb"}},{"code_sha256_prefix":"3f1558360c090b44","entry":"flip_sides","repo":"testingautomated-usi/fashion-mnist-c","repo_kind":"listed","path":"additional_corruptions.py","file_url":"https://github.com/testingautomated-usi/fashion-mnist-c/blob/HEAD/additional_corruptions.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3f1558360c090b44"}},{"code_sha256_prefix":"01cd0fbb90369028","entry":"flip_up_down","repo":"testingautomated-usi/fashion-mnist-c","repo_kind":"listed","path":"additional_corruptions.py","file_url":"https://github.com/testingautomated-usi/fashion-mnist-c/blob/HEAD/additional_corruptions.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"01cd0fbb90369028"}},{"code_sha256_prefix":"20f8669ba3834517","entry":"turn_left","repo":"testingautomated-usi/fashion-mnist-c","repo_kind":"listed","path":"additional_corruptions.py","file_url":"https://github.com/testingautomated-usi/fashion-mnist-c/blob/HEAD/additional_corruptions.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"20f8669ba3834517"}},{"code_sha256_prefix":"712d7aea3437c370","entry":"generate_mix_dataset","repo":"testingautomated-usi/fashion-mnist-c","repo_kind":"listed","path":"generator.py","file_url":"https://github.com/testingautomated-usi/fashion-mnist-c/blob/HEAD/generator.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"712d7aea3437c370"}},{"code_sha256_prefix":"9f67f6434ab46c83","entry":"line_from_points","repo":"testingautomated-usi/fashion-mnist-c","repo_kind":"listed","path":"mnist_c.py","file_url":"https://github.com/testingautomated-usi/fashion-mnist-c/blob/HEAD/mnist_c.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9f67f6434ab46c83"}},{"code_sha256_prefix":"3fe212e00cbe6fb3","entry":"plasma_fractal","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"3fe212e00cbe6fb3"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}