{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/whitening-consistently-improves-self","title":"Whitening Consistently Improves Self-Supervised Learning","arxiv_id":"2408.07519","date":"2024-08-14","proceeding":null,"authors":["András Kalapos","Bálint Gyires-Tóth"],"abstract":"Self-supervised learning (SSL) has been shown to be a powerful approach for learning visual representations. In this study, we propose incorporating ZCA whitening as the final layer of the encoder in self-supervised learning to enhance the quality of learned features by normalizing and decorrelating them. Although whitening has been utilized in SSL in previous works, its potential to universally improve any SSL model has not been explored. We demonstrate that adding whitening as the last layer of SSL pretrained encoders is independent of the self-supervised learning method and encoder architecture, thus it improves performance for a wide range of SSL methods across multiple encoder architectures and datasets. Our experiments show that whitening is capable of improving linear and k-NN probing accuracy by 1-5%. Additionally, we propose metrics that allow for a comprehensive analysis of the learned features, provide insights into the quality of the representations and help identify collapse patterns.","url_abs":"https://arxiv.org/abs/2408.07519v1","url_pdf":"https://arxiv.org/pdf/2408.07519v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"whitening-consistently-improves-self","repo_url":"https://github.com/kaland313/ssl-whitening","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"self-supervised-learning","task_name":"Self-Supervised Learning"}],"methods":[{"method_slug":"zca-whitening","method_name":"ZCA Whitening"},{"method_slug":"k-nn","method_name":"k-NN"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2408.07519","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2408.07519"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kaland313/ssl-whitening","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":1,"unverified":1},"by_repo_kind":{"official":{"samples":2,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"bbd13c06e7baff56","entry":"compute_contrastive_acc","repo":"kaland313/ssl-whitening","repo_kind":"official","path":"pretrain/metrics.py","file_url":"https://github.com/kaland313/ssl-whitening/blob/HEAD/pretrain/metrics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bbd13c06e7baff56"}},{"code_sha256_prefix":"069a72ee985476e3","entry":"eval_feature_descriptors","repo":"kaland313/ssl-whitening","repo_kind":"official","path":"pretrain/metrics.py","file_url":"https://github.com/kaland313/ssl-whitening/blob/HEAD/pretrain/metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"069a72ee985476e3"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}