{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/slicing-vision-transformer-for-flexible","title":"Slicing Vision Transformer for Flexible Inference","arxiv_id":"2412.04786","date":"2024-12-06","proceeding":null,"authors":["Yitian Zhang","Huseyin Coskun","Xu Ma","Huan Wang","Ke Ma","Xi","Chen","Derek Hao Hu","Yun Fu"],"abstract":"Vision Transformers (ViT) is known for its scalability. In this work, we target to scale down a ViT to fit in an environment with dynamic-changing resource constraints. We observe that smaller ViTs are intrinsically the sub-networks of a larger ViT with different widths. Thus, we propose a general framework, named Scala, to enable a single network to represent multiple smaller ViTs with flexible inference capability, which aligns with the inherent design of ViT to vary from widths. Concretely, Scala activates several subnets during training, introduces Isolated Activation to disentangle the smallest sub-network from other subnets, and leverages Scale Coordination to ensure each sub-network receives simplified, steady, and accurate learning objectives. Comprehensive empirical validations on different tasks demonstrate that with only one-shot training, Scala learns slimmable representation without modifying the original ViT structure and matches the performance of Separate Training. Compared with the prior art, Scala achieves an average improvement of 1.6% on ImageNet-1K with fewer parameters.","url_abs":"https://arxiv.org/abs/2412.04786v1","url_pdf":"https://arxiv.org/pdf/2412.04786v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"slicing-vision-transformer-for-flexible","repo_url":"https://github.com/BeSpontaneous/Scala-pytorch","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2412.04786","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2412.04786"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/BeSpontaneous/Scala-pytorch","reach":null}],"summary":{"ran":2,"unverified":8},"by_repo_kind":{"official":{"samples":10,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"7a9dd4ed0208fd2f","entry":"Attention","repo":"BeSpontaneous/Scala-pytorch","repo_kind":"official","path":"models_scala.py","file_url":"https://github.com/BeSpontaneous/Scala-pytorch/blob/HEAD/models_scala.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7a9dd4ed0208fd2f"}},{"code_sha256_prefix":"8264b58dfb638456","entry":"PatchEmbed","repo":"BeSpontaneous/Scala-pytorch","repo_kind":"official","path":"models_scala.py","file_url":"https://github.com/BeSpontaneous/Scala-pytorch/blob/HEAD/models_scala.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8264b58dfb638456"}},{"code_sha256_prefix":"3ab7a1aca590784e","entry":"Block","repo":"BeSpontaneous/Scala-pytorch","repo_kind":"official","path":"models_scala.py","file_url":"https://github.com/BeSpontaneous/Scala-pytorch/blob/HEAD/models_scala.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3ab7a1aca590784e"}},{"code_sha256_prefix":"128790adb097edba","entry":"Format","repo":"BeSpontaneous/Scala-pytorch","repo_kind":"official","path":"models_scala.py","file_url":"https://github.com/BeSpontaneous/Scala-pytorch/blob/HEAD/models_scala.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"128790adb097edba"}},{"code_sha256_prefix":"722ebc6a6b34ca4d","entry":"LayerScale","repo":"BeSpontaneous/Scala-pytorch","repo_kind":"official","path":"models_scala.py","file_url":"https://github.com/BeSpontaneous/Scala-pytorch/blob/HEAD/models_scala.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"722ebc6a6b34ca4d"}},{"code_sha256_prefix":"3a08e0b2bc5ced4b","entry":"Mlp","repo":"BeSpontaneous/Scala-pytorch","repo_kind":"official","path":"models_scala.py","file_url":"https://github.com/BeSpontaneous/Scala-pytorch/blob/HEAD/models_scala.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3a08e0b2bc5ced4b"}},{"code_sha256_prefix":"9040806f1a5cf930","entry":"VisionTransformer_scala","repo":"BeSpontaneous/Scala-pytorch","repo_kind":"official","path":"models_scala.py","file_url":"https://github.com/BeSpontaneous/Scala-pytorch/blob/HEAD/models_scala.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9040806f1a5cf930"}},{"code_sha256_prefix":"dcc5c2638565f2e5","entry":"_assert","repo":"BeSpontaneous/Scala-pytorch","repo_kind":"official","path":"models_scala.py","file_url":"https://github.com/BeSpontaneous/Scala-pytorch/blob/HEAD/models_scala.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dcc5c2638565f2e5"}},{"code_sha256_prefix":"ac707dd1c89e5865","entry":"get_init_weights_vit","repo":"BeSpontaneous/Scala-pytorch","repo_kind":"official","path":"models_scala.py","file_url":"https://github.com/BeSpontaneous/Scala-pytorch/blob/HEAD/models_scala.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ac707dd1c89e5865"}},{"code_sha256_prefix":"38a6c338916f95c8","entry":"init_weights_vit_timm","repo":"BeSpontaneous/Scala-pytorch","repo_kind":"official","path":"models_scala.py","file_url":"https://github.com/BeSpontaneous/Scala-pytorch/blob/HEAD/models_scala.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"38a6c338916f95c8"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}