{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/multiplication-free-transformer-training-via-1","title":"Multiplication-Free Transformer Training via Piecewise Affine Operations","arxiv_id":"2305.17190","date":"2023-05-26","proceeding":"NeurIPS 2023 11","authors":["Atli Kosson","Martin Jaggi"],"abstract":"Multiplications are responsible for most of the computational cost involved in neural network training and inference. Recent research has thus looked for ways to reduce the cost associated with them. Inspired by Mogami (2020), we replace multiplication with a cheap piecewise affine approximation that is achieved by adding the bit representation of the floating point numbers together as integers. We show that transformers can be trained with the resulting modified matrix multiplications on both vision and language tasks with little to no performance impact, and without changes to the training hyperparameters. We further replace all non-linearities in the networks making them fully and jointly piecewise affine in both inputs and weights. Finally, we show that we can eliminate all multiplications in the entire training process, including operations in the forward pass, backward pass and optimizer update, demonstrating the first successful training of modern neural network architectures in a fully multiplication-free fashion.","url_abs":"https://arxiv.org/abs/2305.17190v2","url_pdf":"https://arxiv.org/pdf/2305.17190v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"multiplication-free-transformer-training-via-1","repo_url":"https://github.com/epfml/piecewise-affine-multiplication","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2305.17190","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2305.17190"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/epfml/piecewise-affine-multiplication","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_honours":3,"ran":2,"unverified":14},"by_repo_kind":{"official":{"samples":19,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"410bffb8a07b52e3","entry":"count_params","repo":"epfml/piecewise-affine-multiplication","repo_kind":"official","path":"submodules/timm/benchmark.py","file_url":"https://github.com/epfml/piecewise-affine-multiplication/blob/HEAD/submodules/timm/benchmark.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":2,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"410bffb8a07b52e3"}},{"code_sha256_prefix":"33bca8cbc866bf64","entry":"checkpoint_metric","repo":"epfml/piecewise-affine-multiplication","repo_kind":"official","path":"submodules/timm/avg_checkpoints.py","file_url":"https://github.com/epfml/piecewise-affine-multiplication/blob/HEAD/submodules/timm/avg_checkpoints.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"33bca8cbc866bf64"}},{"code_sha256_prefix":"50d653ee6a59955c","entry":"cuda_timestamp","repo":"epfml/piecewise-affine-multiplication","repo_kind":"official","path":"submodules/timm/benchmark.py","file_url":"https://github.com/epfml/piecewise-affine-multiplication/blob/HEAD/submodules/timm/benchmark.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"50d653ee6a59955c"}},{"code_sha256_prefix":"d4192f5e85770fdd","entry":"find_version","repo":"epfml/piecewise-affine-multiplication","repo_kind":"official","path":"submodules/fairseq/release_utils.py","file_url":"https://github.com/epfml/piecewise-affine-multiplication/blob/HEAD/submodules/fairseq/release_utils.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d4192f5e85770fdd"}},{"code_sha256_prefix":"475160c5a1a1225a","entry":"timestamp","repo":"epfml/piecewise-affine-multiplication","repo_kind":"official","path":"submodules/timm/benchmark.py","file_url":"https://github.com/epfml/piecewise-affine-multiplication/blob/HEAD/submodules/timm/benchmark.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"475160c5a1a1225a"}},{"code_sha256_prefix":"253be6c0ef2fe548","entry":"check_row_major","repo":"epfml/piecewise-affine-multiplication","repo_kind":"official","path":"pam/cuda_bindings.py","file_url":"https://github.com/epfml/piecewise-affine-multiplication/blob/HEAD/pam/cuda_bindings.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"253be6c0ef2fe548"}},{"code_sha256_prefix":"7e941323d268d90d","entry":"cross_entropy","repo":"epfml/piecewise-affine-multiplication","repo_kind":"official","path":"pam/pam_opt.py","file_url":"https://github.com/epfml/piecewise-affine-multiplication/blob/HEAD/pam/pam_opt.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7e941323d268d90d"}},{"code_sha256_prefix":"30174a8ded3bfc64","entry":"demote_tensor","repo":"epfml/piecewise-affine-multiplication","repo_kind":"official","path":"pam/utils.py","file_url":"https://github.com/epfml/piecewise-affine-multiplication/blob/HEAD/pam/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"30174a8ded3bfc64"}},{"code_sha256_prefix":"ab41ea0c2bcafe1e","entry":"div","repo":"epfml/piecewise-affine-multiplication","repo_kind":"official","path":"pam/pam_ops.py","file_url":"https://github.com/epfml/piecewise-affine-multiplication/blob/HEAD/pam/pam_ops.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ab41ea0c2bcafe1e"}},{"code_sha256_prefix":"8414688e931a8025","entry":"exp","repo":"epfml/piecewise-affine-multiplication","repo_kind":"official","path":"pam/pam_ops.py","file_url":"https://github.com/epfml/piecewise-affine-multiplication/blob/HEAD/pam/pam_ops.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8414688e931a8025"}},{"code_sha256_prefix":"88cf61b729ed21dd","entry":"get_exp_mantissa","repo":"epfml/piecewise-affine-multiplication","repo_kind":"official","path":"pam/native.py","file_url":"https://github.com/epfml/piecewise-affine-multiplication/blob/HEAD/pam/native.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"88cf61b729ed21dd"}},{"code_sha256_prefix":"44b6e8848744468f","entry":"get_next_version","repo":"epfml/piecewise-affine-multiplication","repo_kind":"official","path":"submodules/fairseq/release_utils.py","file_url":"https://github.com/epfml/piecewise-affine-multiplication/blob/HEAD/submodules/fairseq/release_utils.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"44b6e8848744468f"}},{"code_sha256_prefix":"a578037399ac5710","entry":"get_norm","repo":"epfml/piecewise-affine-multiplication","repo_kind":"official","path":"shared/modules/normalization.py","file_url":"https://github.com/epfml/piecewise-affine-multiplication/blob/HEAD/shared/modules/normalization.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a578037399ac5710"}},{"code_sha256_prefix":"85332a013577bdd1","entry":"mul","repo":"epfml/piecewise-affine-multiplication","repo_kind":"official","path":"pam/pam_ops.py","file_url":"https://github.com/epfml/piecewise-affine-multiplication/blob/HEAD/pam/pam_ops.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"85332a013577bdd1"}},{"code_sha256_prefix":"aec5df54dd4ceba5","entry":"pa_exp2_fwd","repo":"epfml/piecewise-affine-multiplication","repo_kind":"official","path":"pam/native.py","file_url":"https://github.com/epfml/piecewise-affine-multiplication/blob/HEAD/pam/native.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"aec5df54dd4ceba5"}},{"code_sha256_prefix":"6411768e64113a79","entry":"pa_log2_fwd","repo":"epfml/piecewise-affine-multiplication","repo_kind":"official","path":"pam/native.py","file_url":"https://github.com/epfml/piecewise-affine-multiplication/blob/HEAD/pam/native.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6411768e64113a79"}},{"code_sha256_prefix":"68c33c7d589f43b4","entry":"torch_log_softmax2","repo":"epfml/piecewise-affine-multiplication","repo_kind":"official","path":"pam/pam_experimental.py","file_url":"https://github.com/epfml/piecewise-affine-multiplication/blob/HEAD/pam/pam_experimental.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"68c33c7d589f43b4"}},{"code_sha256_prefix":"3bd7633c5b60762e","entry":"torch_logsumexp2","repo":"epfml/piecewise-affine-multiplication","repo_kind":"official","path":"pam/pam_experimental.py","file_url":"https://github.com/epfml/piecewise-affine-multiplication/blob/HEAD/pam/pam_experimental.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3bd7633c5b60762e"}},{"code_sha256_prefix":"112d967966670abf","entry":"torch_softmax2","repo":"epfml/piecewise-affine-multiplication","repo_kind":"official","path":"pam/pam_experimental.py","file_url":"https://github.com/epfml/piecewise-affine-multiplication/blob/HEAD/pam/pam_experimental.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"112d967966670abf"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}