{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/hyperparameter-ensembles-for-robustness-and","title":"Hyperparameter Ensembles for Robustness and Uncertainty Quantification","arxiv_id":"2006.13570","date":"2020-06-24","proceeding":"NeurIPS 2020 12","authors":["Florian Wenzel","Jasper Snoek","Dustin Tran","Rodolphe Jenatton"],"abstract":"Ensembles over neural network weights trained from different random initialization, known as deep ensembles, achieve state-of-the-art accuracy and calibration. The recently introduced batch ensembles provide a drop-in replacement that is more parameter efficient. In this paper, we design ensembles not only over weights, but over hyperparameters to improve the state of the art in both settings. For best performance independent of budget, we propose hyper-deep ensembles, a simple procedure that involves a random search over different hyperparameters, themselves stratified across multiple random initializations. Its strong performance highlights the benefit of combining models with both weight and hyperparameter diversity. We further propose a parameter efficient version, hyper-batch ensembles, which builds on the layer structure of batch ensembles and self-tuning networks. The computational and memory costs of our method are notably lower than typical ensembles. On image classification tasks, with MLP, LeNet, ResNet 20 and Wide ResNet 28-10 architectures, we improve upon both deep and batch ensembles.","url_abs":"https://arxiv.org/abs/2006.13570v3","url_pdf":"https://arxiv.org/pdf/2006.13570v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"hyperparameter-ensembles-for-robustness-and","repo_url":"https://github.com/google/uncertainty-baselines","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"tf","reach":null},{"paper_slug":"hyperparameter-ensembles-for-robustness-and","repo_url":"https://github.com/google/uncertainty-baselines/blob/master/baselines/cifar/hyperbatchensemble.py","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"tf","reach":null},{"paper_slug":"hyperparameter-ensembles-for-robustness-and","repo_url":"https://github.com/Mind23-2/MindCode-101/tree/main/lenet","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null}],"tasks":[{"task_slug":"diversity","task_name":"Diversity"},{"task_slug":"image-classification","task_name":"Image Classification"},{"task_slug":"uncertainty-quantification","task_name":"Uncertainty Quantification"},{"task_slug":"image-classification","task_name":"image-classification"}],"methods":[{"method_slug":"average-pooling","method_name":"Average Pooling"},{"method_slug":"batch-normalization","method_name":"Batch Normalization"},{"method_slug":"convolution","method_name":"Convolution"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"feedforward-network","method_name":"Feedforward Network"},{"method_slug":"global-average-pooling","method_name":"Global Average Pooling"},{"method_slug":"kaiming-initialization","method_name":"Kaiming Initialization"},{"method_slug":"lenet","method_name":"LeNet"},{"method_slug":"random-search","method_name":"Random Search"},{"method_slug":"relu","method_name":"ReLU"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"wide-residual-block","method_name":"Wide Residual Block"},{"method_slug":"wideresnet","method_name":"WideResNet"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2006.13570","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2006.13570"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/google/edward2","reach":{"status":"unanswered"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/google/uncertainty-baselines","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Mind23-2/MindCode-101/tree/main/lenet","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/google/uncertainty-baselines/blob/master/baselines/cifar/hyperbatchensemble.py","reach":null}],"summary":{"ran":1,"ran_draft_wrong":1,"unverified":2},"by_repo_kind":{"official":{"samples":4,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"109f2e00c01a3c25","entry":"LogScaler","repo":"google/uncertainty-baselines","repo_kind":"official","path":"uncertainty_baselines/models/wide_resnet_hyperbatchensemble.py","file_url":"https://github.com/google/uncertainty-baselines/blob/HEAD/uncertainty_baselines/models/wide_resnet_hyperbatchensemble.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"109f2e00c01a3c25"}},{"code_sha256_prefix":"8c4984235cb0593e","entry":"get_sweep","repo":"google/uncertainty-baselines","repo_kind":"official","path":"baselines/cifar/hyperdeepensemble_configs/cifar10_rand_search_sweep.py","file_url":"https://github.com/google/uncertainty-baselines/blob/HEAD/baselines/cifar/hyperdeepensemble_configs/cifar10_rand_search_sweep.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8c4984235cb0593e"}},{"code_sha256_prefix":"1c87402304aeab38","entry":"make_sign_initializer","repo":"google/uncertainty-baselines","repo_kind":"official","path":"uncertainty_baselines/models/wide_resnet_hyperbatchensemble.py","file_url":"https://github.com/google/uncertainty-baselines/blob/HEAD/uncertainty_baselines/models/wide_resnet_hyperbatchensemble.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1c87402304aeab38"}},{"code_sha256_prefix":"99ee23e47179ff60","entry":"wide_resnet_hyperbatchensemble","repo":"google/uncertainty-baselines","repo_kind":"official","path":"uncertainty_baselines/models/wide_resnet_hyperbatchensemble.py","file_url":"https://github.com/google/uncertainty-baselines/blob/HEAD/uncertainty_baselines/models/wide_resnet_hyperbatchensemble.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"99ee23e47179ff60"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}