{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/small-models-struggle-to-learn-from-strong","title":"Small Models Struggle to Learn from Strong Reasoners","arxiv_id":"2502.12143","date":"2025-02-17","proceeding":null,"authors":["Yuetai Li","Xiang Yue","Zhangchen Xu","Fengqing Jiang","Luyao Niu","Bill Yuchen Lin","Bhaskar Ramasubramanian","Radha Poovendran"],"abstract":"Large language models (LLMs) excel in complex reasoning tasks, and distilling their reasoning capabilities into smaller models has shown promise. However, we uncover an interesting phenomenon, which we term the Small Model Learnability Gap: small models ($\\leq$3B parameters) do not consistently benefit from long chain-of-thought (CoT) reasoning or distillation from larger models. Instead, they perform better when fine-tuned on shorter, simpler reasoning chains that better align with their intrinsic learning capacity. To address this, we propose Mix Distillation, a simple yet effective strategy that balances reasoning complexity by combining long and short CoT examples or reasoning from both larger and smaller models. Our experiments demonstrate that Mix Distillation significantly improves small model reasoning performance compared to training on either data alone. These findings highlight the limitations of direct strong model distillation and underscore the importance of adapting reasoning complexity for effective reasoning capability transfer.","url_abs":"https://arxiv.org/abs/2502.12143v1","url_pdf":"https://arxiv.org/pdf/2502.12143v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"small-models-struggle-to-learn-from-strong","repo_url":"https://github.com/Small-Model-Gap/Small-Model-Learnability-Gap","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[],"methods":[{"method_slug":"align","method_name":"ALIGN"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2502.12143","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2502.12143"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/Small-Model-Gap/Small-Model-Learnability-Gap","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":1,"unverified":7},"by_repo_kind":{"listed":{"samples":8,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"20a7cc804eb22661","entry":"hash_args","repo":"Small-Model-Gap/Small-Model-Learnability-Gap","repo_kind":"listed","path":"lm-evaluation-harness/lm_eval/api/model.py","file_url":"https://github.com/Small-Model-Gap/Small-Model-Learnability-Gap/blob/HEAD/lm-evaluation-harness/lm_eval/api/model.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"20a7cc804eb22661"}},{"code_sha256_prefix":"f39c1ddb182ba813","entry":"apply_liger_kernel_to_qwen2_5_vl","repo":"Small-Model-Gap/Small-Model-Learnability-Gap","repo_kind":"listed","path":"LLaMA-Factory/src/llamafactory/model/model_utils/liger_kernel.py","file_url":"https://github.com/Small-Model-Gap/Small-Model-Learnability-Gap/blob/HEAD/LLaMA-Factory/src/llamafactory/model/model_utils/liger_kernel.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f39c1ddb182ba813"}},{"code_sha256_prefix":"30c7c5d6b0c5a0b1","entry":"get_custom_gradient_checkpointing_func","repo":"Small-Model-Gap/Small-Model-Learnability-Gap","repo_kind":"listed","path":"LLaMA-Factory/src/llamafactory/model/model_utils/checkpointing.py","file_url":"https://github.com/Small-Model-Gap/Small-Model-Learnability-Gap/blob/HEAD/LLaMA-Factory/src/llamafactory/model/model_utils/checkpointing.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"30c7c5d6b0c5a0b1"}},{"code_sha256_prefix":"a3a4aba09f5afb8b","entry":"llama_attention_forward","repo":"Small-Model-Gap/Small-Model-Learnability-Gap","repo_kind":"listed","path":"LLaMA-Factory/src/llamafactory/model/model_utils/longlora.py","file_url":"https://github.com/Small-Model-Gap/Small-Model-Learnability-Gap/blob/HEAD/LLaMA-Factory/src/llamafactory/model/model_utils/longlora.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a3a4aba09f5afb8b"}},{"code_sha256_prefix":"638d2764440f23dc","entry":"llama_flash_attention_2_forward","repo":"Small-Model-Gap/Small-Model-Learnability-Gap","repo_kind":"listed","path":"LLaMA-Factory/src/llamafactory/model/model_utils/longlora.py","file_url":"https://github.com/Small-Model-Gap/Small-Model-Learnability-Gap/blob/HEAD/LLaMA-Factory/src/llamafactory/model/model_utils/longlora.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"638d2764440f23dc"}},{"code_sha256_prefix":"897cfc7be8a5bfd1","entry":"llama_sdpa_attention_forward","repo":"Small-Model-Gap/Small-Model-Learnability-Gap","repo_kind":"listed","path":"LLaMA-Factory/src/llamafactory/model/model_utils/longlora.py","file_url":"https://github.com/Small-Model-Gap/Small-Model-Learnability-Gap/blob/HEAD/LLaMA-Factory/src/llamafactory/model/model_utils/longlora.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"897cfc7be8a5bfd1"}},{"code_sha256_prefix":"03670fe07fee5b3f","entry":"load_config","repo":"Small-Model-Gap/Small-Model-Learnability-Gap","repo_kind":"listed","path":"LLaMA-Factory/src/llamafactory/model/loader.py","file_url":"https://github.com/Small-Model-Gap/Small-Model-Learnability-Gap/blob/HEAD/LLaMA-Factory/src/llamafactory/model/loader.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"03670fe07fee5b3f"}},{"code_sha256_prefix":"9dcc10d9409f0809","entry":"patch_valuehead_model","repo":"Small-Model-Gap/Small-Model-Learnability-Gap","repo_kind":"listed","path":"LLaMA-Factory/src/llamafactory/model/patcher.py","file_url":"https://github.com/Small-Model-Gap/Small-Model-Learnability-Gap/blob/HEAD/LLaMA-Factory/src/llamafactory/model/patcher.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9dcc10d9409f0809"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}