{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2606-03498","title":"Demystifying Pipeline Parallelism: First Theory for PipeDream","arxiv_id":"2606.03498","date":"2026-06-02","proceeding":null,"authors":["Ivan Ilin","Peter Richtárik"],"abstract":"Training modern machine learning models increasingly requires computation to be distributed across many accelerators. Data parallelism remains the default choice and is often paired with tensor-parallel sharding, but model parallelism becomes unavoidable once parameters, activations, or optimizer states no longer fit on a single device. This paper studies pipeline model parallelism through the lens of PipeDream (PD) (Harlap et al., 2018). Our first contribution is theoretical: we introduce Randomized PipeDream (RPD), a stale block-SGD abstraction that yields, to our knowledge, the first clean nonconvex convergence guarantee for a PD-style method. Our second contribution is a scaling diagnosis: we prove that the delay induced by steady-state PD grows as $S^2 - S/2 + O(1)$ for $S$ stages, so the stale-read contribution in the convergence theorem scales as $Θ(γ^2 S^4)$, equivalently as $Θ(S^4/K)$ in the tuned-rate form. Our third contribution is a comparison with LocalSGD, whose periodic model averaging trades weight staleness for synchronization bubbles. In our reported simulated-time experiments, the better-performing method depends on the objective: PD performs better on the quadratic objective and on a small language-modeling training-loss task, while for logistic regression LocalSGD becomes superior as the number of stages increases.","url_abs":"https://arxiv.org/abs/2606.03498","url_pdf":"https://arxiv.org/pdf/2606.03498","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2606.03498","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2606.03498"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/vectozavr/randomized-pipedream","reach":null}],"summary":{"ran":8},"by_repo_kind":{"found_in_text":{"samples":8,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":8,"samples":[{"code_sha256_prefix":"628afb9b004058e3","entry":"build_pipedream_exact_delays","repo":"vectozavr/randomized-pipedream","repo_kind":"found_in_text","path":"src/methods/gpd.py","file_url":"https://github.com/vectozavr/randomized-pipedream/blob/HEAD/src/methods/gpd.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"628afb9b004058e3"}},{"code_sha256_prefix":"6c31c0bac3d26860","entry":"build_prediction_from_stage_weights","repo":"vectozavr/randomized-pipedream","repo_kind":"found_in_text","path":"src/methods/utils.py","file_url":"https://github.com/vectozavr/randomized-pipedream/blob/HEAD/src/methods/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6c31c0bac3d26860"}},{"code_sha256_prefix":"61f1177a1afa653e","entry":"count_backward_ops","repo":"vectozavr/randomized-pipedream","repo_kind":"found_in_text","path":"llm_experiments.py","file_url":"https://github.com/vectozavr/randomized-pipedream/blob/HEAD/llm_experiments.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"61f1177a1afa653e"}},{"code_sha256_prefix":"2a9be104e190740a","entry":"extract_pipedream_backward_ops","repo":"vectozavr/randomized-pipedream","repo_kind":"found_in_text","path":"src/methods/gpd.py","file_url":"https://github.com/vectozavr/randomized-pipedream/blob/HEAD/src/methods/gpd.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"2a9be104e190740a"}},{"code_sha256_prefix":"17a18ca59575d99c","entry":"find_optimal_microbatches","repo":"vectozavr/randomized-pipedream","repo_kind":"found_in_text","path":"llm_pd_vs_sgd.py","file_url":"https://github.com/vectozavr/randomized-pipedream/blob/HEAD/llm_pd_vs_sgd.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"17a18ca59575d99c"}},{"code_sha256_prefix":"f06f9d2a19155c36","entry":"parse_int_list","repo":"vectozavr/randomized-pipedream","repo_kind":"found_in_text","path":"llm_experiments.py","file_url":"https://github.com/vectozavr/randomized-pipedream/blob/HEAD/llm_experiments.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f06f9d2a19155c36"}},{"code_sha256_prefix":"04842bba1a60e7e2","entry":"parse_lr_grid","repo":"vectozavr/randomized-pipedream","repo_kind":"found_in_text","path":"llm_experiments.py","file_url":"https://github.com/vectozavr/randomized-pipedream/blob/HEAD/llm_experiments.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"04842bba1a60e7e2"}},{"code_sha256_prefix":"40781d29307b9c5e","entry":"sample_stale_read_indices","repo":"vectozavr/randomized-pipedream","repo_kind":"found_in_text","path":"src/methods/utils.py","file_url":"https://github.com/vectozavr/randomized-pipedream/blob/HEAD/src/methods/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"40781d29307b9c5e"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.LG","source":"arxiv_2026.jsonl"},"syntology_extracted_results":null}