{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/second-order-forward-mode-automatic","title":"Second-Order Forward-Mode Automatic Differentiation for Optimization","arxiv_id":"2408.10419","date":"2024-08-19","proceeding":null,"authors":["Adam D. Cobb","Atılım Güneş Baydin","Barak A. Pearlmutter","Susmit Jha"],"abstract":"This paper introduces a second-order hyperplane search, a novel optimization step that generalizes a second-order line search from a line to a $k$-dimensional hyperplane. This, combined with the forward-mode stochastic gradient method, yields a second-order optimization algorithm that consists of forward passes only, completely avoiding the storage overhead of backpropagation. Unlike recent work that relies on directional derivatives (or Jacobian--Vector Products, JVPs), we use hyper-dual numbers to jointly evaluate both directional derivatives and their second-order quadratic terms. As a result, we introduce forward-mode weight perturbation with Hessian information (FoMoH). We then use FoMoH to develop a novel generalization of line search by extending it to a hyperplane search. We illustrate the utility of this extension and how it might be used to overcome some of the recent challenges of optimizing machine learning models without backpropagation. Our code is open-sourced at https://github.com/SRI-CSL/fomoh.","url_abs":"https://arxiv.org/abs/2408.10419v1","url_pdf":"https://arxiv.org/pdf/2408.10419v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"second-order-forward-mode-automatic","repo_url":"https://github.com/sri-csl/fomoh","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"BSD-2-Clause"}}],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2408.10419","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2408.10419"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sri-csl/fomoh","reach":{"status":"ok","spdx":"BSD-2-Clause"}}],"summary":{"ran":10},"by_repo_kind":{"official":{"samples":10,"ran":10,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"03c4d3b2fd4d59f6","entry":"calculate_output_shape","repo":"sri-csl/fomoh","repo_kind":"official","path":"src/fomoh/util.py","file_url":"https://github.com/sri-csl/fomoh/blob/HEAD/src/fomoh/util.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"03c4d3b2fd4d59f6"}},{"code_sha256_prefix":"13430b547d6472d9","entry":"compute_d2Z_dT1T2","repo":"sri-csl/fomoh","repo_kind":"official","path":"src/fomoh/util.py","file_url":"https://github.com/sri-csl/fomoh/blob/HEAD/src/fomoh/util.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"13430b547d6472d9"}},{"code_sha256_prefix":"5c67ce0d094b0545","entry":"compute_dZ_dT","repo":"sri-csl/fomoh","repo_kind":"official","path":"src/fomoh/util.py","file_url":"https://github.com/sri-csl/fomoh/blob/HEAD/src/fomoh/util.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"5c67ce0d094b0545"}},{"code_sha256_prefix":"83f36c715570b676","entry":"init_bias","repo":"sri-csl/fomoh","repo_kind":"official","path":"src/fomoh/layers.py","file_url":"https://github.com/sri-csl/fomoh/blob/HEAD/src/fomoh/layers.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"83f36c715570b676"}},{"code_sha256_prefix":"aeb04885a2579d96","entry":"init_kaiming","repo":"sri-csl/fomoh","repo_kind":"official","path":"src/fomoh/layers.py","file_url":"https://github.com/sri-csl/fomoh/blob/HEAD/src/fomoh/layers.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"aeb04885a2579d96"}},{"code_sha256_prefix":"38ee06cbca66bf07","entry":"init_uniform","repo":"sri-csl/fomoh","repo_kind":"official","path":"src/fomoh/layers.py","file_url":"https://github.com/sri-csl/fomoh/blob/HEAD/src/fomoh/layers.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"38ee06cbca66bf07"}},{"code_sha256_prefix":"e60ee540637c5ca4","entry":"make_tensor","repo":"sri-csl/fomoh","repo_kind":"official","path":"src/fomoh/hyperdual.py","file_url":"https://github.com/sri-csl/fomoh/blob/HEAD/src/fomoh/hyperdual.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"e60ee540637c5ca4"}},{"code_sha256_prefix":"68aa86b28a69f9b3","entry":"mean","repo":"sri-csl/fomoh","repo_kind":"official","path":"src/fomoh/nn.py","file_url":"https://github.com/sri-csl/fomoh/blob/HEAD/src/fomoh/nn.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"68aa86b28a69f9b3"}},{"code_sha256_prefix":"f3958881d36885b5","entry":"nll_loss","repo":"sri-csl/fomoh","repo_kind":"official","path":"src/fomoh/nn.py","file_url":"https://github.com/sri-csl/fomoh/blob/HEAD/src/fomoh/nn.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"f3958881d36885b5"}},{"code_sha256_prefix":"f5019e843d4e2762","entry":"nll_loss_old","repo":"sri-csl/fomoh","repo_kind":"official","path":"src/fomoh/nn.py","file_url":"https://github.com/sri-csl/fomoh/blob/HEAD/src/fomoh/nn.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-2-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"f5019e843d4e2762"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}