{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/quasi-hyperbolic-momentum-and-adam-for-deep","title":"Quasi-hyperbolic momentum and Adam for deep learning","arxiv_id":"1810.06801","date":"2018-10-16","proceeding":"ICLR 2019 5","authors":["Jerry Ma","Denis Yarats"],"abstract":"Momentum-based acceleration of stochastic gradient descent (SGD) is widely\nused in deep learning. We propose the quasi-hyperbolic momentum algorithm (QHM)\nas an extremely simple alteration of momentum SGD, averaging a plain SGD step\nwith a momentum step. We describe numerous connections to and identities with\nother algorithms, and we characterize the set of two-state optimization\nalgorithms that QHM can recover. Finally, we propose a QH variant of Adam\ncalled QHAdam, and we empirically demonstrate that our algorithms lead to\nsignificantly improved training in a variety of settings, including a new\nstate-of-the-art result on WMT16 EN-DE. We hope that these empirical results,\ncombined with the conceptual and practical simplicity of QHM and QHAdam, will\nspur interest from both practitioners and researchers. Code is immediately\navailable.","url_abs":"http://arxiv.org/abs/1810.06801v4","url_pdf":"http://arxiv.org/pdf/1810.06801v4.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"quasi-hyperbolic-momentum-and-adam-for-deep","repo_url":"https://github.com/facebookresearch/qhoptim","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"quasi-hyperbolic-momentum-and-adam-for-deep","repo_url":"https://github.com/sajadn/QHAdam","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}}],"tasks":[{"task_slug":"deep-learning","task_name":"Deep Learning"},{"task_slug":"stochastic-optimization","task_name":"Stochastic Optimization"}],"methods":[{"method_slug":"qhadam","method_name":"QHAdam"},{"method_slug":"qhm","method_name":"QHM"},{"method_slug":"sgd","method_name":"SGD"}],"datasets_introduced":[],"methods_introduced":[{"slug":"qhadam","name":"QHAdam","full_name":"QHAdam"},{"slug":"qhm","name":"QHM","full_name":"QHM"}],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/1810.06801","atlas_url":"https://app.syntology.ai/?focus=1810.06801","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1810.06801"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sajadn/QHAdam","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/qhoptim","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":7},"by_repo_kind":{"official":{"samples":7,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":7,"samples":[{"code_sha256_prefix":"e6d8e5bc3f47df76","entry":"QHAdamW","repo":"facebookresearch/qhoptim","repo_kind":"official","path":"qhoptim/pyt/qhadam.py","file_url":"https://github.com/facebookresearch/qhoptim/blob/HEAD/qhoptim/pyt/qhadam.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"e6d8e5bc3f47df76"}},{"code_sha256_prefix":"5597447a89943bbe","entry":"allclose","repo":"facebookresearch/qhoptim","repo_kind":"official","path":"test_qhoptim/tf/util.py","file_url":"https://github.com/facebookresearch/qhoptim/blob/HEAD/test_qhoptim/tf/util.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"5597447a89943bbe"}},{"code_sha256_prefix":"8c61c8f36e03fd81","entry":"build_net","repo":"facebookresearch/qhoptim","repo_kind":"official","path":"test_qhoptim/tf/util.py","file_url":"https://github.com/facebookresearch/qhoptim/blob/HEAD/test_qhoptim/tf/util.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"8c61c8f36e03fd81"}},{"code_sha256_prefix":"f70286c6efb24e49","entry":"call_if_callable","repo":"facebookresearch/qhoptim","repo_kind":"official","path":"qhoptim/tf/util.py","file_url":"https://github.com/facebookresearch/qhoptim/blob/HEAD/qhoptim/tf/util.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"f70286c6efb24e49"}},{"code_sha256_prefix":"85d5bd9c96b637ab","entry":"from_pid","repo":"facebookresearch/qhoptim","repo_kind":"official","path":"qhoptim/common/param_conv.py","file_url":"https://github.com/facebookresearch/qhoptim/blob/HEAD/qhoptim/common/param_conv.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"85d5bd9c96b637ab"}},{"code_sha256_prefix":"f7a58c6038a0e8fa","entry":"from_robust_momentum","repo":"facebookresearch/qhoptim","repo_kind":"official","path":"qhoptim/common/param_conv.py","file_url":"https://github.com/facebookresearch/qhoptim/blob/HEAD/qhoptim/common/param_conv.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"f7a58c6038a0e8fa"}},{"code_sha256_prefix":"e8d42db83273329b","entry":"from_synthesized_nesterov","repo":"facebookresearch/qhoptim","repo_kind":"official","path":"qhoptim/common/param_conv.py","file_url":"https://github.com/facebookresearch/qhoptim/blob/HEAD/qhoptim/common/param_conv.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":false,"mcp_get_code":{"code_sha256":"e8d42db83273329b"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}