{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/a-theoretical-analysis-of-catastrophic","title":"A Theoretical Analysis of Catastrophic Forgetting through the NTK Overlap Matrix","arxiv_id":"2010.04003","date":"2020-10-07","proceeding":null,"authors":["Thang Doan","Mehdi Bennani","Bogdan Mazoure","Guillaume Rabusseau","Pierre Alquier"],"abstract":"Continual learning (CL) is a setting in which an agent has to learn from an incoming stream of data during its entire lifetime. Although major advances have been made in the field, one recurring problem which remains unsolved is that of Catastrophic Forgetting (CF). While the issue has been extensively studied empirically, little attention has been paid from a theoretical angle. In this paper, we show that the impact of CF increases as two tasks increasingly align. We introduce a measure of task similarity called the NTK overlap matrix which is at the core of CF. We analyze common projected gradient algorithms and demonstrate how they mitigate forgetting. Then, we propose a variant of Orthogonal Gradient Descent (OGD) which leverages structure of the data through Principal Component Analysis (PCA). Experiments support our theoretical findings and show how our method can help reduce CF on classical CL datasets.","url_abs":"https://arxiv.org/abs/2010.04003v2","url_pdf":"https://arxiv.org/pdf/2010.04003v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"a-theoretical-analysis-of-catastrophic","repo_url":"https://github.com/tldoan/PCA-OGD","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"a-theoretical-analysis-of-catastrophic","repo_url":"https://github.com/azizanlab/sketchogd","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"continual-learning","task_name":"Continual Learning"}],"methods":[{"method_slug":"3d-convolution","method_name":"3D Convolution"},{"method_slug":"ntk","method_name":"NTK"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2010.04003","atlas_url":"https://app.syntology.ai/?focus=2010.04003","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.04003"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/azizanlab/sketchogd","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/tldoan/PCA-OGD","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":13},"by_repo_kind":{"official":{"samples":13,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"b5b0b299677171a5","entry":"CIFAR10","repo":"tldoan/PCA-OGD","repo_kind":"official","path":"dataloaders/base.py","file_url":"https://github.com/tldoan/PCA-OGD/blob/HEAD/dataloaders/base.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b5b0b299677171a5"}},{"code_sha256_prefix":"0d51ab16aa5bc90c","entry":"CIFAR100","repo":"tldoan/PCA-OGD","repo_kind":"official","path":"dataloaders/base.py","file_url":"https://github.com/tldoan/PCA-OGD/blob/HEAD/dataloaders/base.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0d51ab16aa5bc90c"}},{"code_sha256_prefix":"270673c93e064a59","entry":"LeNetC","repo":"tldoan/PCA-OGD","repo_kind":"official","path":"models/lenet.py","file_url":"https://github.com/tldoan/PCA-OGD/blob/HEAD/models/lenet.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"270673c93e064a59"}},{"code_sha256_prefix":"72c9598b309e4e5a","entry":"MNIST","repo":"tldoan/PCA-OGD","repo_kind":"official","path":"dataloaders/base.py","file_url":"https://github.com/tldoan/PCA-OGD/blob/HEAD/dataloaders/base.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"72c9598b309e4e5a"}},{"code_sha256_prefix":"f9b8caa6279ce116","entry":"PermutedGen","repo":"tldoan/PCA-OGD","repo_kind":"official","path":"dataloaders/datasetGen.py","file_url":"https://github.com/tldoan/PCA-OGD/blob/HEAD/dataloaders/datasetGen.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f9b8caa6279ce116"}},{"code_sha256_prefix":"25d61b37abe1981b","entry":"RotatedGen","repo":"tldoan/PCA-OGD","repo_kind":"official","path":"dataloaders/datasetGen.py","file_url":"https://github.com/tldoan/PCA-OGD/blob/HEAD/dataloaders/datasetGen.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"25d61b37abe1981b"}},{"code_sha256_prefix":"8c9829ef771c8fe8","entry":"SplitGen","repo":"tldoan/PCA-OGD","repo_kind":"official","path":"dataloaders/datasetGen.py","file_url":"https://github.com/tldoan/PCA-OGD/blob/HEAD/dataloaders/datasetGen.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8c9829ef771c8fe8"}},{"code_sha256_prefix":"25decb913a86724b","entry":"accuracy","repo":"tldoan/PCA-OGD","repo_kind":"official","path":"utils/utils.py","file_url":"https://github.com/tldoan/PCA-OGD/blob/HEAD/utils/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"25decb913a86724b"}},{"code_sha256_prefix":"5dd8f3694051c1e0","entry":"orthonormalize","repo":"tldoan/PCA-OGD","repo_kind":"official","path":"algos/ogd.py","file_url":"https://github.com/tldoan/PCA-OGD/blob/HEAD/algos/ogd.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5dd8f3694051c1e0"}},{"code_sha256_prefix":"62c3f89b98633488","entry":"penalty","repo":"tldoan/PCA-OGD","repo_kind":"official","path":"algos/ewc.py","file_url":"https://github.com/tldoan/PCA-OGD/blob/HEAD/algos/ewc.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"62c3f89b98633488"}},{"code_sha256_prefix":"25f23f7771ae8223","entry":"project2cone2","repo":"tldoan/PCA-OGD","repo_kind":"official","path":"algos/gem.py","file_url":"https://github.com/tldoan/PCA-OGD/blob/HEAD/algos/gem.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"25f23f7771ae8223"}},{"code_sha256_prefix":"1db8a5e0aca11e0d","entry":"project_vec","repo":"tldoan/PCA-OGD","repo_kind":"official","path":"algos/ogd.py","file_url":"https://github.com/tldoan/PCA-OGD/blob/HEAD/algos/ogd.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1db8a5e0aca11e0d"}},{"code_sha256_prefix":"6b253a0b38ed156a","entry":"test_error","repo":"tldoan/PCA-OGD","repo_kind":"official","path":"utils/utils.py","file_url":"https://github.com/tldoan/PCA-OGD/blob/HEAD/utils/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6b253a0b38ed156a"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}