{"url":"/method/gradient-sparsification","slug":"gradient-sparsification","name":"Gradient Sparsification","full_name":"Gradient Sparsification","full_name_withheld":false,"description_markdown":"**Gradient Sparsification** is a technique for distributed training that sparsifies stochastic gradients to reduce the communication cost, with minor increase in the number of iterations. The key idea behind our sparsification technique is to drop some coordinates of the stochastic gradient and appropriately amplify the remaining coordinates to ensure the unbiasedness of the sparsified stochastic gradient. The sparsification approach can significantly reduce the coding length of the stochastic gradient and only slightly increase the variance of the stochastic gradient.","description_state":"present","introduced_year":null,"introduced_by":{"title":"Gradient Sparsification for Communication-Efficient Distributed Optimization","paper":"/paper/gradient-sparsification-for-communication","first_author":"Jianqiao Wangni","n_authors":4,"url_abs":null,"archive_paper_url":"https://paperswithcode.com/paper/gradient-sparsification-for-communication"},"source":{"url":"http://arxiv.org/abs/1710.09854v1","title":"Gradient Sparsification for Communication-Efficient Distributed Optimization","url_on_a_paper_host":true},"code_snippet_url":null,"code_snippet_url_on_a_code_host":false,"categories":[{"area":"General","area_id":"general","collection":"Data Parallel Methods","url":"/methods/category/data-parallel-methods","pwc_aliases":[]},{"area":"General","area_id":"general","collection":"Stochastic Optimization","url":"/methods/category/stochastic-optimization","pwc_aliases":[]},{"area":"General","area_id":"general","collection":"Optimization","url":"/methods/category/optimization","pwc_aliases":[]},{"area":"General","area_id":"general","collection":"Distributed Methods","url":"/methods/category/distributed-methods","pwc_aliases":[]}],"n_papers_tagged":38,"archive_num_papers":38,"papers_newest_first":[{"paper":null,"title":"Mobility-Aware Asynchronous Federated Learning with Dynamic Sparsification","date":"2025-06-08","arxiv_id":"2506.07328","n_code_links":0,"syntology":null},{"paper":"/paper/dynamic-gradient-sparsification-training-for","title":"Dynamic Gradient Sparsification Training for Few-Shot Fine-tuning of CT Lymph Node Segmentation Foundation Model","date":"2025-03-02","arxiv_id":"2503.00748","n_code_links":1,"syntology":null},{"paper":null,"title":"Sparse Incremental Aggregation in Satellite Federated Learning","date":"2025-01-20","arxiv_id":"2501.11385","n_code_links":0,"syntology":null},{"paper":null,"title":"Regularized Top-$k$: A Bayesian Framework for Gradient Sparsification","date":"2025-01-10","arxiv_id":"2501.05633","n_code_links":0,"syntology":null},{"paper":"/paper/dqrm-deep-quantized-recommendation-models","title":"DQRM: Deep Quantized Recommendation Models","date":"2024-10-26","arxiv_id":"2410.20046","n_code_links":1,"syntology":null},{"paper":null,"title":"Age-of-Gradient Updates for Federated Learning over Random Access Channels","date":"2024-10-15","arxiv_id":"2410.11986","n_code_links":0,"syntology":null},{"paper":null,"title":"Novel Gradient Sparsification Algorithm via Bayesian Inference","date":"2024-09-23","arxiv_id":"2409.14893","n_code_links":0,"syntology":null},{"paper":"/paper/preserving-near-optimal-gradient","title":"Preserving Near-Optimal Gradient Sparsification Cost for Scalable Distributed Deep Learning","date":"2024-02-21","arxiv_id":"2402.13781","n_code_links":1,"syntology":null},{"paper":"/paper/jointsq-joint-sparsification-quantization-for","title":"JointSQ: Joint Sparsification-Quantization for Distributed Learning","date":"2024-01-01","arxiv_id":null,"n_code_links":1,"syntology":null},{"paper":null,"title":"RS-DGC: Exploring Neighborhood Statistics for Dynamic Gradient Compression on Remote Sensing Image Interpretation","date":"2023-12-29","arxiv_id":"2312.17530","n_code_links":0,"syntology":null},{"paper":"/paper/micro-near-zero-cost-gradient-sparsification","title":"MiCRO: Near-Zero Cost Gradient Sparsification for Scaling and Accelerating Distributed DNN Training","date":"2023-10-02","arxiv_id":"2310.00967","n_code_links":1,"syntology":null},{"paper":null,"title":"Gradient Sparsification For Masked Fine-Tuning of Transformers","date":"2023-07-19","arxiv_id":"2307.10098","n_code_links":0,"syntology":null},{"paper":"/paper/deft-exploiting-gradient-norm-difference","title":"DEFT: Exploiting Gradient Norm Difference between Model Layers for Scalable Gradient Sparsification","date":"2023-07-07","arxiv_id":"2307.03500","n_code_links":1,"syntology":null},{"paper":null,"title":"Gradient Sparsification for Efficient Wireless Federated Learning with Differential Privacy","date":"2023-04-09","arxiv_id":"2304.04164","n_code_links":0,"syntology":null},{"paper":null,"title":"Efficient and Secure Federated Learning for Financial Applications","date":"2023-03-15","arxiv_id":"2303.08355","n_code_links":0,"syntology":null},{"paper":null,"title":"On the Interaction Between Differential Privacy and Gradient Compression in Deep Learning","date":"2022-11-01","arxiv_id":"2211.00734","n_code_links":0,"syntology":null},{"paper":null,"title":"Downlink Compression Improves TopK Sparsification","date":"2022-09-30","arxiv_id":"2209.15203","n_code_links":0,"syntology":null},{"paper":null,"title":"Empirical Analysis on Top-k Gradient Sparsification for Distributed Deep Learning in a Supercomputing Environment","date":"2022-09-18","arxiv_id":"2209.08497","n_code_links":0,"syntology":null},{"paper":"/paper/near-optimal-sparse-allreduce-for-distributed","title":"Near-Optimal Sparse Allreduce for Distributed Deep Learning","date":"2022-01-19","arxiv_id":"2201.07598","n_code_links":1,"syntology":null},{"paper":null,"title":"Sparsified Secure Aggregation for Privacy-Preserving Federated Learning","date":"2021-12-23","arxiv_id":"2112.12872","n_code_links":0,"syntology":null},{"paper":null,"title":"Communication-Efficient Federated Learning via Quantized Compressed Sensing","date":"2021-11-30","arxiv_id":"2111.15071","n_code_links":0,"syntology":null},{"paper":null,"title":"Federated Dynamic Neural Network for Deep MIMO Detection","date":"2021-11-24","arxiv_id":"2111.12260","n_code_links":0,"syntology":null},{"paper":null,"title":"Gradient Sparsification For \\emph{Masked Fine-Tuning} of Transformers","date":"2021-11-16","arxiv_id":null,"n_code_links":0,"syntology":null},{"paper":null,"title":"Linear Convergence in Federated Learning: Tackling Client Heterogeneity and Sparse Gradients","date":"2021-02-14","arxiv_id":"2102.07053","n_code_links":0,"syntology":null},{"paper":null,"title":"Large-Scale Training System for 100-Million Classification at Alibaba","date":"2021-02-09","arxiv_id":"2102.06025","n_code_links":0,"syntology":null},{"paper":null,"title":"Gradient Sparsification Can Improve Performance of Differentially-Private Convex Machine Learning","date":"2020-11-30","arxiv_id":"2011.14572","n_code_links":0,"syntology":null},{"paper":"/paper/towards-building-a-robust-and-fair-federated","title":"A Reputation Mechanism Is All You Need: Collaborative Fairness and Adversarial Robustness in Federated Learning","date":"2020-11-20","arxiv_id":"2011.10464","n_code_links":2,"syntology":null},{"paper":"/paper/flame-differentially-private-federated","title":"FLAME: Differentially Private Federated Learning in the Shuffle Model","date":"2020-09-17","arxiv_id":"2009.08063","n_code_links":1,"syntology":{"ran":0,"of":1,"unverified":1,"pointer_only":0}},{"paper":null,"title":"rTop-k: A Statistical Estimation Approach to Distributed SGD","date":"2020-05-21","arxiv_id":"2005.10761","n_code_links":0,"syntology":null},{"paper":null,"title":"Adaptive Gradient Sparsification for Efficient Federated Learning: An Online Learning Approach","date":"2020-01-14","arxiv_id":"2001.04756","n_code_links":0,"syntology":null}],"papers_shown":30,"tasks":[{"task":"/task/federated-learning","name":"Federated Learning","papers":13},{"task":"/task/quantization","name":"Quantization","papers":5},{"task":"/task/deep-learning","name":"Deep Learning","papers":3},{"task":"/task/distributed-optimization","name":"Distributed Optimization","papers":3},{"task":"/task/stochastic-optimization","name":"Stochastic Optimization","papers":3},{"task":"/task/machine-learning","name":"BIG-bench Machine Learning","papers":2},{"task":"/task/fairness","name":"Fairness","papers":2},{"task":null,"name":"GPU","papers":2},{"task":"/task/privacy-preserving","name":"Privacy Preserving","papers":2},{"task":"/task/transfer-learning","name":"Transfer Learning","papers":2},{"task":"/task/compressed-sensing","name":"compressed sensing","papers":2},{"task":"/task/adversarial-defense","name":"Adversarial Defense","papers":1},{"task":"/task/adversarial-robustness","name":"Adversarial Robustness","papers":1},{"task":"/task/all","name":"All","papers":1},{"task":"/task/bayesian-inference","name":"Bayesian Inference","papers":1},{"task":"/task/binarization","name":"Binarization","papers":1},{"task":"/task/classification-1","name":"Classification","papers":1},{"task":"/task/collaborative-fairness","name":"Collaborative Fairness","papers":1},{"task":"/task/earth-observation","name":"Earth Observation","papers":1},{"task":"/task/classification","name":"General Classification","papers":1}],"tasks_shown":20,"n_tasks":28,"usage_by_year":[{"year":"2017","papers":1},{"year":"2018","papers":3},{"year":"2019","papers":4},{"year":"2020","papers":5},{"year":"2021","papers":6},{"year":"2022","papers":4},{"year":"2023","papers":6},{"year":"2024","papers":5},{"year":"2025","papers":4}],"row_source":"methods_table","archive":{"source":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","archive_url":"https://paperswithcode.com/method/gradient-sparsification"},"syntology_read_at":"2026-09-24T18:15:14+00:00"}