{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/stochastic-gradient-descent-with-hyperbolic","title":"Stochastic Gradient Descent with Hyperbolic-Tangent Decay on Classification","arxiv_id":"1806.01593","date":"2018-06-05","proceeding":null,"authors":["Bo Yang Hsueh","Wei Li","I-Chen Wu"],"abstract":"Learning rate scheduler has been a critical issue in the deep neural network\ntraining. Several schedulers and methods have been proposed, including step\ndecay scheduler, adaptive method, cosine scheduler and cyclical scheduler. This\npaper proposes a new scheduling method, named hyperbolic-tangent decay (HTD).\nWe run experiments on several benchmarks such as: ResNet, Wide ResNet and\nDenseNet for CIFAR-10 and CIFAR-100 datasets, LSTM for PAMAP2 dataset, ResNet\non ImageNet and Fashion-MNIST datasets. In our experiments, HTD outperforms\nstep decay and cosine scheduler in nearly all cases, while requiring less\nhyperparameters than step decay, and more flexible than cosine scheduler. Code\nis available at https://github.com/BIGBALLON/HTD.","url_abs":"http://arxiv.org/abs/1806.01593v2","url_pdf":"http://arxiv.org/pdf/1806.01593v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"stochastic-gradient-descent-with-hyperbolic","repo_url":"https://github.com/BIGBALLON/HTD","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"stochastic-gradient-descent-with-hyperbolic","repo_url":"https://github.com/karthikRavichandran/LR_exploration_tf","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok"}},{"paper_slug":"stochastic-gradient-descent-with-hyperbolic","repo_url":"https://github.com/MindSpore-scientific-2/code-5/tree/main/cyclic-cosine-decay-master","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null},{"paper_slug":"stochastic-gradient-descent-with-hyperbolic","repo_url":"https://github.com/MindSpore-scientific/code-4/tree/main/cyclic-cosine-decay-master","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null},{"paper_slug":"stochastic-gradient-descent-with-hyperbolic","repo_url":"https://github.com/pwc-1/Paper-9/tree/main/4/cyclic-cosine-decay-master","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null}],"tasks":[{"task_slug":"classification-1","task_name":"Classification"},{"task_slug":"classification","task_name":"General Classification"},{"task_slug":"scheduling","task_name":"Scheduling"}],"methods":[{"method_slug":"1x1-convolution","method_name":"1x1 Convolution"},{"method_slug":"average-pooling","method_name":"Average Pooling"},{"method_slug":"batch-normalization","method_name":"Batch Normalization"},{"method_slug":"bottleneck-residual-block","method_name":"Bottleneck Residual Block"},{"method_slug":"convolution","method_name":"Convolution"},{"method_slug":"global-average-pooling","method_name":"Global Average Pooling"},{"method_slug":"kaiming-initialization","method_name":"Kaiming Initialization"},{"method_slug":"lstm","method_name":"LSTM"},{"method_slug":"max-pooling","method_name":"Max Pooling"},{"method_slug":"relu","method_name":"ReLU"},{"method_slug":"residual-block","method_name":"Residual Block"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"sigmoid-activation","method_name":"Sigmoid Activation"},{"method_slug":"tanh-activation","method_name":"Tanh Activation"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":null,"mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}