{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/large-language-models-are-reasoning-teachers","title":"Large Language Models Are Reasoning Teachers","arxiv_id":"2212.10071","date":"2022-12-20","proceeding":null,"authors":["Namgyu Ho","Laura Schmid","Se-Young Yun"],"abstract":"Recent works have shown that chain-of-thought (CoT) prompting can elicit language models to solve complex reasoning tasks, step-by-step. However, prompt-based CoT methods are dependent on very large models such as GPT-3 175B which are prohibitive to deploy at scale. In this paper, we use these large models as reasoning teachers to enable complex reasoning in smaller models and reduce model size requirements by several orders of magnitude. We propose Fine-tune-CoT, a method that generates reasoning samples from very large teacher models to fine-tune smaller models. We evaluate our method on a wide range of public models and complex tasks. We find that Fine-tune-CoT enables substantial reasoning capability in small models, far outperforming prompt-based baselines and even the teacher model in many tasks. Additionally, we extend our method by leveraging the teacher model's ability to generate multiple distinct rationales for each original sample. Enriching the fine-tuning data with such diverse reasoning results in a substantial performance boost across datasets, even for very small models. We conduct ablations and sample studies to understand the emergence of reasoning capabilities of student models. Our code implementation and data are available at https://github.com/itsnamgyu/reasoning-teacher.","url_abs":"https://arxiv.org/abs/2212.10071v2","url_pdf":"https://arxiv.org/pdf/2212.10071v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"large-language-models-are-reasoning-teachers","repo_url":"https://github.com/itsnamgyu/reasoning-teacher","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"cosine-annealing","method_name":"Cosine Annealing"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt-3","method_name":"GPT-3"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-cosine-annealing","method_name":"Linear Warmup With Cosine Annealing"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2212.10071","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2212.10071"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/itsnamgyu/reasoning-teacher","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":9},"by_repo_kind":{"official":{"samples":9,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"342c80134c65f9ad","entry":"get_completion_data_path","repo":"itsnamgyu/reasoning-teacher","repo_kind":"official","path":"src/paths.py","file_url":"https://github.com/itsnamgyu/reasoning-teacher/blob/HEAD/src/paths.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"342c80134c65f9ad"}},{"code_sha256_prefix":"3df12d92fd1ba3ec","entry":"get_dataset_path","repo":"itsnamgyu/reasoning-teacher","repo_kind":"official","path":"src/paths.py","file_url":"https://github.com/itsnamgyu/reasoning-teacher/blob/HEAD/src/paths.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3df12d92fd1ba3ec"}},{"code_sha256_prefix":"2a6bb42a8d2676fc","entry":"get_json_value","repo":"itsnamgyu/reasoning-teacher","repo_kind":"official","path":"src/oai/utils/metadata.py","file_url":"https://github.com/itsnamgyu/reasoning-teacher/blob/HEAD/src/oai/utils/metadata.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2a6bb42a8d2676fc"}},{"code_sha256_prefix":"5cc7cc5521428f77","entry":"get_model_key","repo":"itsnamgyu/reasoning-teacher","repo_kind":"official","path":"src/oai/utils/metadata.py","file_url":"https://github.com/itsnamgyu/reasoning-teacher/blob/HEAD/src/oai/utils/metadata.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5cc7cc5521428f77"}},{"code_sha256_prefix":"c2bc595db0df3786","entry":"get_split_path","repo":"itsnamgyu/reasoning-teacher","repo_kind":"official","path":"src/paths.py","file_url":"https://github.com/itsnamgyu/reasoning-teacher/blob/HEAD/src/paths.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c2bc595db0df3786"}},{"code_sha256_prefix":"c715cc1e173d5091","entry":"get_token_count","repo":"itsnamgyu/reasoning-teacher","repo_kind":"official","path":"src/oai/utils/tokens.py","file_url":"https://github.com/itsnamgyu/reasoning-teacher/blob/HEAD/src/oai/utils/tokens.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c715cc1e173d5091"}},{"code_sha256_prefix":"72d8d5514149d7da","entry":"list_of_dicts_to_dict_of_lists","repo":"itsnamgyu/reasoning-teacher","repo_kind":"official","path":"src/custom/utils.py","file_url":"https://github.com/itsnamgyu/reasoning-teacher/blob/HEAD/src/custom/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"72d8d5514149d7da"}},{"code_sha256_prefix":"3fea59ef4d0740f4","entry":"summarize_evaluation","repo":"itsnamgyu/reasoning-teacher","repo_kind":"official","path":"src/evaluation/summary.py","file_url":"https://github.com/itsnamgyu/reasoning-teacher/blob/HEAD/src/evaluation/summary.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3fea59ef4d0740f4"}},{"code_sha256_prefix":"faa49eca817c0635","entry":"truncate_by_n_tokens","repo":"itsnamgyu/reasoning-teacher","repo_kind":"official","path":"src/oai/utils/tokens.py","file_url":"https://github.com/itsnamgyu/reasoning-teacher/blob/HEAD/src/oai/utils/tokens.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"faa49eca817c0635"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}