{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/decentralized-deep-learning-with-arbitrary","title":"Decentralized Deep Learning with Arbitrary Communication Compression","arxiv_id":"1907.09356","date":"2019-07-22","proceeding":"ICLR 2020 1","authors":["Anastasia Koloskova","Tao Lin","Sebastian U. Stich","Martin Jaggi"],"abstract":"Decentralized training of deep learning models is a key element for enabling data privacy and on-device learning over networks, as well as for efficient scaling to large compute clusters. As current approaches suffer from limited bandwidth of the network, we propose the use of communication compression in the decentralized training context. We show that Choco-SGD $-$ recently introduced and analyzed for strongly-convex objectives only $-$ converges under arbitrary high compression ratio on general non-convex functions at the rate $O\\bigl(1/\\sqrt{nT}\\bigr)$ where $T$ denotes the number of iterations and $n$ the number of workers. The algorithm achieves linear speedup in the number of workers and supports higher compression than previous state-of-the art methods. We demonstrate the practical performance of the algorithm in two key scenarios: the training of deep learning models (i) over distributed user devices, connected by a social network and (ii) in a datacenter (outperforming all-reduce time-wise).","url_abs":"https://arxiv.org/abs/1907.09356v3","url_pdf":"https://arxiv.org/pdf/1907.09356v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"decentralized-deep-learning-with-arbitrary","repo_url":"https://github.com/epfml/ChocoSGD","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"deep-learning","task_name":"Deep Learning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=1907.09356","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1907.09356"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/epfml/ChocoSGD","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_violates":1,"unverified":4},"by_repo_kind":{"official":{"samples":5,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"f017532fc389cbfe","entry":"str2bool","repo":"epfml/ChocoSGD","repo_kind":"official","path":"dl_code/parameters.py","file_url":"https://github.com/epfml/ChocoSGD/blob/HEAD/dl_code/parameters.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f017532fc389cbfe"}},{"code_sha256_prefix":"ec7de8d73be07d34","entry":"load_data_batch","repo":"epfml/ChocoSGD","repo_kind":"official","path":"dl_code/pcode/create_dataset.py","file_url":"https://github.com/epfml/ChocoSGD/blob/HEAD/dl_code/pcode/create_dataset.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ec7de8d73be07d34"}},{"code_sha256_prefix":"a1f434c84fe53743","entry":"qsgd_quantize","repo":"epfml/ChocoSGD","repo_kind":"official","path":"convex_code/logistic.py","file_url":"https://github.com/epfml/ChocoSGD/blob/HEAD/convex_code/logistic.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a1f434c84fe53743"}},{"code_sha256_prefix":"438c8b99e9634673","entry":"to_percent","repo":"epfml/ChocoSGD","repo_kind":"official","path":"convex_code/utils.py","file_url":"https://github.com/epfml/ChocoSGD/blob/HEAD/convex_code/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"438c8b99e9634673"}},{"code_sha256_prefix":"70bc7a292e413fb6","entry":"unpickle_dir","repo":"epfml/ChocoSGD","repo_kind":"official","path":"convex_code/utils.py","file_url":"https://github.com/epfml/ChocoSGD/blob/HEAD/convex_code/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"70bc7a292e413fb6"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}