{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/dytox-transformers-for-continual-learning","title":"DyTox: Transformers for Continual Learning with DYnamic TOken eXpansion","arxiv_id":"2111.11326","date":"2021-11-22","proceeding":"CVPR 2022 1","authors":["Arthur Douillard","Alexandre Ramé","Guillaume Couairon","Matthieu Cord"],"abstract":"Deep network architectures struggle to continually learn new tasks without forgetting the previous tasks. A recent trend indicates that dynamic architectures based on an expansion of the parameters can reduce catastrophic forgetting efficiently in continual learning. However, existing approaches often require a task identifier at test-time, need complex tuning to balance the growing number of parameters, and barely share any information across tasks. As a result, they struggle to scale to a large number of tasks without significant overhead. In this paper, we propose a transformer architecture based on a dedicated encoder/decoder framework. Critically, the encoder and decoder are shared among all tasks. Through a dynamic expansion of special tokens, we specialize each forward of our decoder network on a task distribution. Our strategy scales to a large number of tasks while having negligible memory and time overheads due to strict control of the parameters expansion. Moreover, this efficient strategy doesn't need any hyperparameter tuning to control the network's expansion. Our model reaches excellent results on CIFAR100 and state-of-the-art performances on the large-scale ImageNet100 and ImageNet1000 while having less parameters than concurrent dynamic frameworks.","url_abs":"https://arxiv.org/abs/2111.11326v3","url_pdf":"https://arxiv.org/pdf/2111.11326v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"dytox-transformers-for-continual-learning","repo_url":"https://github.com/arthurdouillard/dytox","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"class-incremental-learning","task_name":"Class Incremental Learning"},{"task_slug":"continual-learning","task_name":"Continual Learning"},{"task_slug":"decoder","task_name":"Decoder"},{"task_slug":"incremental-learning","task_name":"Incremental Learning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/incremental-learning-on-imagenet-10-steps","task":"Incremental Learning","dataset":"ImageNet - 10 steps","model":"DyTox","rank_in_archive_order":2,"of":10,"metrics":{"# M Params":"11.36","Average Incremental Accuracy":"71.29","Average Incremental Accuracy Top-5":"88.59","Final Accuracy":"63.34","Final Accuracy Top-5":"84.49"},"uses_additional_data":false},{"leaderboard":"/sota/incremental-learning-on-imagenet100-10-steps","task":"Incremental Learning","dataset":"ImageNet100 - 10 steps","model":"DyTox","rank_in_archive_order":7,"of":13,"metrics":{"# M Params":"11.01","Average Incremental Accuracy":"77.15","Average Incremental Accuracy Top-5":"92.04","Final Accuracy":"69.10","Final Accuracy Top-5":"87.98"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2111.11326","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2111.11326"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/arthurdouillard/dytox","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_draft_wrong":2,"ran_violates":1,"unverified":8},"by_repo_kind":{"official":{"samples":11,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":11,"samples":[{"code_sha256_prefix":"2a80220dabcb742a","entry":"conv1x1","repo":"arthurdouillard/dytox","repo_kind":"official","path":"continual/cnn/resnet.py","file_url":"https://github.com/arthurdouillard/dytox/blob/HEAD/continual/cnn/resnet.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"2a80220dabcb742a"}},{"code_sha256_prefix":"600ff2c45e0de056","entry":"conv3x3","repo":"arthurdouillard/dytox","repo_kind":"official","path":"continual/cnn/resnet.py","file_url":"https://github.com/arthurdouillard/dytox/blob/HEAD/continual/cnn/resnet.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"600ff2c45e0de056"}},{"code_sha256_prefix":"d00cae076f919bb2","entry":"eval_training_finetuning","repo":"arthurdouillard/dytox","repo_kind":"official","path":"continual/dytox.py","file_url":"https://github.com/arthurdouillard/dytox/blob/HEAD/continual/dytox.py","link_basis":"plan_row","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"d00cae076f919bb2"}},{"code_sha256_prefix":"1d2fde37d898b3fe","entry":"bce_smooth_pos_with_logits","repo":"arthurdouillard/dytox","repo_kind":"official","path":"continual/losses.py","file_url":"https://github.com/arthurdouillard/dytox/blob/HEAD/continual/losses.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"1d2fde37d898b3fe"}},{"code_sha256_prefix":"7457ba60be827516","entry":"bce_with_logits","repo":"arthurdouillard/dytox","repo_kind":"official","path":"continual/losses.py","file_url":"https://github.com/arthurdouillard/dytox/blob/HEAD/continual/losses.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"7457ba60be827516"}},{"code_sha256_prefix":"ad865c09bd36571e","entry":"build_transform","repo":"arthurdouillard/dytox","repo_kind":"official","path":"continual/datasets.py","file_url":"https://github.com/arthurdouillard/dytox/blob/HEAD/continual/datasets.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"ad865c09bd36571e"}},{"code_sha256_prefix":"7d9b83035fd371d5","entry":"conv1x1","repo":"arthurdouillard/dytox","repo_kind":"official","path":"continual/cnn/resnet_scs.py","file_url":"https://github.com/arthurdouillard/dytox/blob/HEAD/continual/cnn/resnet_scs.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"7d9b83035fd371d5"}},{"code_sha256_prefix":"d4448dd8427a42bd","entry":"conv3x3","repo":"arthurdouillard/dytox","repo_kind":"official","path":"continual/cnn/resnet_scs.py","file_url":"https://github.com/arthurdouillard/dytox/blob/HEAD/continual/cnn/resnet_scs.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"d4448dd8427a42bd"}},{"code_sha256_prefix":"7c129ba97c54ef15","entry":"forward","repo":"arthurdouillard/dytox","repo_kind":"official","path":"continual/engine.py","file_url":"https://github.com/arthurdouillard/dytox/blob/HEAD/continual/engine.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"7c129ba97c54ef15"}},{"code_sha256_prefix":"66aa6d9bb27298d3","entry":"soft_bce_with_logits","repo":"arthurdouillard/dytox","repo_kind":"official","path":"continual/losses.py","file_url":"https://github.com/arthurdouillard/dytox/blob/HEAD/continual/losses.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"66aa6d9bb27298d3"}},{"code_sha256_prefix":"71abd18f83eed1c9","entry":"unfold2d","repo":"arthurdouillard/dytox","repo_kind":"official","path":"continual/cnn/resnet_scs.py","file_url":"https://github.com/arthurdouillard/dytox/blob/HEAD/continual/cnn/resnet_scs.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":false,"mcp_get_code":{"code_sha256":"71abd18f83eed1c9"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}