{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/text-transformer-2-self-adaptive-llms","title":"Transformer-Squared: Self-adaptive LLMs","arxiv_id":"2501.06252","date":"2025-01-09","proceeding":null,"authors":["Qi Sun","Edoardo Cetin","Yujin Tang"],"abstract":"Self-adaptive large language models (LLMs) aim to solve the challenges posed by traditional fine-tuning methods, which are often computationally intensive and static in their ability to handle diverse tasks. We introduce Transformer-Squared, a novel self-adaptation framework that adapts LLMs for unseen tasks in real-time by selectively adjusting only the singular components of their weight matrices. During inference, Transformer-Squared employs a two-pass mechanism: first, a dispatch system identifies the task properties, and then task-specific 'expert' vectors, trained using reinforcement learning, are dynamically mixed to obtain targeted behavior for the incoming prompt. Our method consistently outperforms ubiquitous approaches such as LoRA, with fewer parameters and greater efficiency. Furthermore, Transformer-Squared demonstrates versatility across different LLM architectures and modalities, including vision-language tasks. Transformer-Squared represents a significant leap forward, offering a scalable, efficient solution for enhancing the adaptability and task-specific performance of LLMs, paving the way for truly dynamic, self-organizing AI systems.","url_abs":"https://arxiv.org/abs/2501.06252v3","url_pdf":"https://arxiv.org/pdf/2501.06252v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"text-transformer-2-self-adaptive-llms","repo_url":"https://github.com/SakanaAI/self-adaptive-llms","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"text-transformer-2-self-adaptive-llms","repo_url":"https://github.com/codelion/adaptive-classifier","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2501.06252","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2501.06252"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/SakanaAI/self-adaptive-llms","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/codelion/adaptive-classifier","reach":null}],"summary":{"ran":1,"unverified":9},"by_repo_kind":{"official":{"samples":8,"ran":0,"repositories":1},"listed":{"samples":2,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"7e31232bbbaca78a","entry":"StrategicOptimizer","repo":"codelion/adaptive-classifier","repo_kind":"listed","path":"src/adaptive_classifier/strategic.py","file_url":"https://github.com/codelion/adaptive-classifier/blob/HEAD/src/adaptive_classifier/strategic.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7e31232bbbaca78a"}},{"code_sha256_prefix":"141e2782e1b569c5","entry":"StrategicCostFunction","repo":"codelion/adaptive-classifier","repo_kind":"listed","path":"src/adaptive_classifier/strategic.py","file_url":"https://github.com/codelion/adaptive-classifier/blob/HEAD/src/adaptive_classifier/strategic.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"141e2782e1b569c5"}},{"code_sha256_prefix":"ebfe6f7380b4fae8","entry":"compose_new_params","repo":"SakanaAI/self-adaptive-llms","repo_kind":"official","path":"utils.py","file_url":"https://github.com/SakanaAI/self-adaptive-llms/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ebfe6f7380b4fae8"}},{"code_sha256_prefix":"dde83d263378109c","entry":"eval_model","repo":"SakanaAI/self-adaptive-llms","repo_kind":"official","path":"utils.py","file_url":"https://github.com/SakanaAI/self-adaptive-llms/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"dde83d263378109c"}},{"code_sha256_prefix":"86a430f986c7a911","entry":"extract_ans","repo":"SakanaAI/self-adaptive-llms","repo_kind":"official","path":"tasks/cls.py","file_url":"https://github.com/SakanaAI/self-adaptive-llms/blob/HEAD/tasks/cls.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"86a430f986c7a911"}},{"code_sha256_prefix":"5154a9cbc712e1a5","entry":"forward","repo":"SakanaAI/self-adaptive-llms","repo_kind":"official","path":"utils.py","file_url":"https://github.com/SakanaAI/self-adaptive-llms/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5154a9cbc712e1a5"}},{"code_sha256_prefix":"c4b88e2b851a7bb0","entry":"get_mean_std_max_min_dict","repo":"SakanaAI/self-adaptive-llms","repo_kind":"official","path":"logging_utils.py","file_url":"https://github.com/SakanaAI/self-adaptive-llms/blob/HEAD/logging_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c4b88e2b851a7bb0"}},{"code_sha256_prefix":"9e3bd956686c6370","entry":"get_soft_mask","repo":"SakanaAI/self-adaptive-llms","repo_kind":"official","path":"policy/base.py","file_url":"https://github.com/SakanaAI/self-adaptive-llms/blob/HEAD/policy/base.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9e3bd956686c6370"}},{"code_sha256_prefix":"d00162523239ac86","entry":"mean","repo":"SakanaAI/self-adaptive-llms","repo_kind":"official","path":"tasks/cls.py","file_url":"https://github.com/SakanaAI/self-adaptive-llms/blob/HEAD/tasks/cls.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d00162523239ac86"}},{"code_sha256_prefix":"7c89a602fda6eb4a","entry":"wandb_init","repo":"SakanaAI/self-adaptive-llms","repo_kind":"official","path":"svd_reinforce_hydra.py","file_url":"https://github.com/SakanaAI/self-adaptive-llms/blob/HEAD/svd_reinforce_hydra.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7c89a602fda6eb4a"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}