{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/stochastic-gradient-push-for-distributed-deep","title":"Stochastic Gradient Push for Distributed Deep Learning","arxiv_id":"1811.10792","date":"2018-11-27","proceeding":"ICLR 2019 5","authors":["Mahmoud Assran","Nicolas Loizou","Nicolas Ballas","Michael Rabbat"],"abstract":"Distributed data-parallel algorithms aim to accelerate the training of deep neural networks by parallelizing the computation of large mini-batch gradient updates across multiple nodes. Approaches that synchronize nodes using exact distributed averaging (e.g., via AllReduce) are sensitive to stragglers and communication delays. The PushSum gossip algorithm is robust to these issues, but only performs approximate distributed averaging. This paper studies Stochastic Gradient Push (SGP), which combines PushSum with stochastic gradient updates. We prove that SGP converges to a stationary point of smooth, non-convex objectives at the same sub-linear rate as SGD, and that all nodes achieve consensus. We empirically validate the performance of SGP on image classification (ResNet-50, ImageNet) and machine translation (Transformer, WMT'16 En-De) workloads. Our code will be made publicly available.","url_abs":"https://arxiv.org/abs/1811.10792v3","url_pdf":"https://arxiv.org/pdf/1811.10792v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"stochastic-gradient-push-for-distributed-deep","repo_url":"https://github.com/VhalPurohit/290s","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"stochastic-gradient-push-for-distributed-deep","repo_url":"https://github.com/adelnabli/acid","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"stochastic-gradient-push-for-distributed-deep","repo_url":"https://github.com/facebookresearch/stochastic_gradient_push","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"deep-learning","task_name":"Deep Learning"},{"task_slug":"classification","task_name":"General Classification"},{"task_slug":"image-classification","task_name":"Image Classification"},{"task_slug":"machine-translation","task_name":"Machine Translation"},{"task_slug":"translation","task_name":"Translation"},{"task_slug":"image-classification","task_name":"image-classification"}],"methods":[{"method_slug":"sgd","method_name":"SGD"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1811.10792","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1811.10792"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/adelnabli/acid","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/facebookresearch/stochastic_gradient_push","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/VhalPurohit/290s","reach":null}],"summary":{"ran_honours":1,"unverified":14},"by_repo_kind":{"listed":{"samples":15,"ran":1,"repositories":3}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":2,"samples":[{"code_sha256_prefix":"ac9e01d5a81fc218","entry":"accuracy","repo":"VhalPurohit/290s","repo_kind":"listed","path":"gossip_sgd.py","file_url":"https://github.com/VhalPurohit/290s/blob/HEAD/gossip_sgd.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"ac9e01d5a81fc218"}},{"code_sha256_prefix":"5a80b1ca6739b208","entry":"accuracy","repo":"facebookresearch/stochastic_gradient_push","repo_kind":"listed","path":"gossip_sgd.py","file_url":"https://github.com/facebookresearch/stochastic_gradient_push/blob/HEAD/gossip_sgd.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NOASSERTION","inline_ok":false,"mcp_get_code":{"code_sha256":"5a80b1ca6739b208"}},{"code_sha256_prefix":"c5377f5726a15290","entry":"add_weight_decay","repo":"adelnabli/acid","repo_kind":"listed","path":"utils/net_utils.py","file_url":"https://github.com/adelnabli/acid/blob/HEAD/utils/net_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c5377f5726a15290"}},{"code_sha256_prefix":"14e905ad950e92b5","entry":"compute_algebraic_connectivity","repo":"adelnabli/acid","repo_kind":"listed","path":"utils/graph_utils.py","file_url":"https://github.com/adelnabli/acid/blob/HEAD/utils/graph_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"14e905ad950e92b5"}},{"code_sha256_prefix":"6fc8a02c499df11c","entry":"compute_graph_resistance","repo":"adelnabli/acid","repo_kind":"listed","path":"utils/graph_utils.py","file_url":"https://github.com/adelnabli/acid/blob/HEAD/utils/graph_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6fc8a02c499df11c"}},{"code_sha256_prefix":"58454b216bad91b4","entry":"compute_laplacian","repo":"adelnabli/acid","repo_kind":"listed","path":"utils/graph_utils.py","file_url":"https://github.com/adelnabli/acid/blob/HEAD/utils/graph_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"58454b216bad91b4"}},{"code_sha256_prefix":"11c7e3c59a515397","entry":"compute_multiplicative_coef_lr","repo":"adelnabli/acid","repo_kind":"listed","path":"utils/net_utils.py","file_url":"https://github.com/adelnabli/acid/blob/HEAD/utils/net_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"11c7e3c59a515397"}},{"code_sha256_prefix":"6f82f2364c4b825c","entry":"create_dict_result","repo":"adelnabli/acid","repo_kind":"listed","path":"utils/logs_utils.py","file_url":"https://github.com/adelnabli/acid/blob/HEAD/utils/logs_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6f82f2364c4b825c"}},{"code_sha256_prefix":"165c8ba03b498eae","entry":"create_model","repo":"adelnabli/acid","repo_kind":"listed","path":"utils/net_utils.py","file_url":"https://github.com/adelnabli/acid/blob/HEAD/utils/net_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"165c8ba03b498eae"}},{"code_sha256_prefix":"553014d88e2fdc2b","entry":"data_loader","repo":"adelnabli/acid","repo_kind":"listed","path":"utils/data_utils.py","file_url":"https://github.com/adelnabli/acid/blob/HEAD/utils/data_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"553014d88e2fdc2b"}},{"code_sha256_prefix":"61f497b58bf5182d","entry":"gather_previous_data","repo":"adelnabli/acid","repo_kind":"listed","path":"utils/logs_utils.py","file_url":"https://github.com/adelnabli/acid/blob/HEAD/utils/logs_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"61f497b58bf5182d"}},{"code_sha256_prefix":"f7ac7056fa97da5c","entry":"init_momentum_var","repo":"adelnabli/acid","repo_kind":"listed","path":"utils/acid_utils.py","file_url":"https://github.com/adelnabli/acid/blob/HEAD/utils/acid_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f7ac7056fa97da5c"}},{"code_sha256_prefix":"465e03a81b60ff44","entry":"print_training_evolution","repo":"adelnabli/acid","repo_kind":"listed","path":"utils/logs_utils.py","file_url":"https://github.com/adelnabli/acid/blob/HEAD/utils/logs_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"465e03a81b60ff44"}},{"code_sha256_prefix":"685ef5ebed3e922f","entry":"test_data","repo":"adelnabli/acid","repo_kind":"listed","path":"utils/data_utils.py","file_url":"https://github.com/adelnabli/acid/blob/HEAD/utils/data_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"685ef5ebed3e922f"}},{"code_sha256_prefix":"b5814cb3e05c8509","entry":"train_data","repo":"adelnabli/acid","repo_kind":"listed","path":"utils/data_utils.py","file_url":"https://github.com/adelnabli/acid/blob/HEAD/utils/data_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b5814cb3e05c8509"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}