{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/solar-10-7b-scaling-large-language-models","title":"SOLAR 10.7B: Scaling Large Language Models with Simple yet Effective Depth Up-Scaling","arxiv_id":"2312.15166","date":"2023-12-23","proceeding":null,"authors":["Dahyun Kim","Chanjun Park","Sanghoon Kim","Wonsung Lee","Wonho Song","Yunsu Kim","Hyeonwoo Kim","Yungi Kim","Hyeonju Lee","Jihoo Kim","Changbae Ahn","Seonghoon Yang","Sukyung Lee","Hyunbyung Park","Gyoungjin Gim","Mikyoung Cha","Hwalsuk Lee","Sunghun Kim"],"abstract":"We introduce SOLAR 10.7B, a large language model (LLM) with 10.7 billion parameters, demonstrating superior performance in various natural language processing (NLP) tasks. Inspired by recent efforts to efficiently up-scale LLMs, we present a method for scaling LLMs called depth up-scaling (DUS), which encompasses depthwise scaling and continued pretraining. In contrast to other LLM up-scaling methods that use mixture-of-experts, DUS does not require complex changes to train and inference efficiently. We show experimentally that DUS is simple yet effective in scaling up high-performance LLMs from small ones. Building on the DUS model, we additionally present SOLAR 10.7B-Instruct, a variant fine-tuned for instruction-following capabilities, surpassing Mixtral-8x7B-Instruct. SOLAR 10.7B is publicly available under the Apache 2.0 license, promoting broad access and application in the LLM field.","url_abs":"https://arxiv.org/abs/2312.15166v3","url_pdf":"https://arxiv.org/pdf/2312.15166v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"solar-10-7b-scaling-large-language-models","repo_url":"https://github.com/jquesnelle/scaled-rope","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"solar-10-7b-scaling-large-language-models","repo_url":"https://github.com/jquesnelle/yarn","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"instruction-following","task_name":"Instruction Following"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"large-language-model","task_name":"Large Language Model"},{"task_slug":"mixture-of-experts","task_name":"Mixture-of-Experts"}],"methods":[{"method_slug":"base","method_name":"BASE"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2312.15166","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.15166"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jquesnelle/yarn","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jquesnelle/scaled-rope","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_fixture":3,"ran":15,"ran_draft_wrong":2,"unverified":6},"by_repo_kind":{"listed":{"samples":26,"ran":20,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"30d7eec482ebf6b1","entry":"repeat_kv","repo":"jquesnelle/scaled-rope","repo_kind":"listed","path":"scaled_rope/modeling_llama_yarn.py","file_url":"https://github.com/jquesnelle/scaled-rope/blob/HEAD/scaled_rope/modeling_llama_yarn.py","link_basis":"plan_row","language":"python","status":"ran_fixture","verification_level":2,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"30d7eec482ebf6b1"}},{"code_sha256_prefix":"cf585714e4ffcb75","entry":"add_args","repo":"jquesnelle/yarn","repo_kind":"listed","path":"eval/model_loader.py","file_url":"https://github.com/jquesnelle/yarn/blob/HEAD/eval/model_loader.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cf585714e4ffcb75"}},{"code_sha256_prefix":"1b69ac244c9fd326","entry":"apply_patches","repo":"jquesnelle/yarn","repo_kind":"listed","path":"eval/model_loader.py","file_url":"https://github.com/jquesnelle/yarn/blob/HEAD/eval/model_loader.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1b69ac244c9fd326"}},{"code_sha256_prefix":"d61c483a3c2b3156","entry":"apply_rotary_pos_emb","repo":"jquesnelle/scaled-rope","repo_kind":"listed","path":"scaled_rope/modeling_llama_yarn.py","file_url":"https://github.com/jquesnelle/scaled-rope/blob/HEAD/scaled_rope/modeling_llama_yarn.py","link_basis":"plan_row","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d61c483a3c2b3156"}},{"code_sha256_prefix":"37fc3129c3d06e0c","entry":"apply_rotary_pos_emb","repo":"jquesnelle/scaled-rope","repo_kind":"listed","path":"scaled_rope/modeling_mistral_yarn.py","file_url":"https://github.com/jquesnelle/scaled-rope/blob/HEAD/scaled_rope/modeling_mistral_yarn.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"37fc3129c3d06e0c"}},{"code_sha256_prefix":"70ceb95d97957b6c","entry":"compute_perplexity","repo":"jquesnelle/yarn","repo_kind":"listed","path":"eval/perplexity.py","file_url":"https://github.com/jquesnelle/yarn/blob/HEAD/eval/perplexity.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"70ceb95d97957b6c"}},{"code_sha256_prefix":"82635a5a905ad930","entry":"find_all_linear_names","repo":"jquesnelle/yarn","repo_kind":"listed","path":"finetune.py","file_url":"https://github.com/jquesnelle/yarn/blob/HEAD/finetune.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"82635a5a905ad930"}},{"code_sha256_prefix":"3c92c963bb0cbb73","entry":"find_correction_dim","repo":"jquesnelle/scaled-rope","repo_kind":"listed","path":"scaled_rope/LlamaDynamicYaRNScaledRotaryEmbedding.py","file_url":"https://github.com/jquesnelle/scaled-rope/blob/HEAD/scaled_rope/LlamaDynamicYaRNScaledRotaryEmbedding.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3c92c963bb0cbb73"}},{"code_sha256_prefix":"191c81a96b555436","entry":"find_correction_factor","repo":"jquesnelle/scaled-rope","repo_kind":"listed","path":"scaled_rope/FalconDynamicPartNTKScaledRotaryEmbedding.py","file_url":"https://github.com/jquesnelle/scaled-rope/blob/HEAD/scaled_rope/FalconDynamicPartNTKScaledRotaryEmbedding.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"191c81a96b555436"}},{"code_sha256_prefix":"763ecb554c17d2fe","entry":"find_correction_range","repo":"jquesnelle/scaled-rope","repo_kind":"listed","path":"scaled_rope/LlamaDynamicYaRNScaledRotaryEmbedding.py","file_url":"https://github.com/jquesnelle/scaled-rope/blob/HEAD/scaled_rope/LlamaDynamicYaRNScaledRotaryEmbedding.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"763ecb554c17d2fe"}},{"code_sha256_prefix":"b4a38f39f9288869","entry":"find_correction_range","repo":"jquesnelle/scaled-rope","repo_kind":"listed","path":"scaled_rope/FalconDynamicPartNTKScaledRotaryEmbedding.py","file_url":"https://github.com/jquesnelle/scaled-rope/blob/HEAD/scaled_rope/FalconDynamicPartNTKScaledRotaryEmbedding.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b4a38f39f9288869"}},{"code_sha256_prefix":"82b8d6fd7a1152e0","entry":"generate_prompt","repo":"jquesnelle/yarn","repo_kind":"listed","path":"eval/passkey.py","file_url":"https://github.com/jquesnelle/yarn/blob/HEAD/eval/passkey.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"82b8d6fd7a1152e0"}},{"code_sha256_prefix":"b21ca20b3500c55f","entry":"generate_prompt","repo":"jquesnelle/yarn","repo_kind":"listed","path":"eval/passkey_hard.py","file_url":"https://github.com/jquesnelle/yarn/blob/HEAD/eval/passkey_hard.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b21ca20b3500c55f"}},{"code_sha256_prefix":"8d8ff31fdde07297","entry":"get_prompt","repo":"jquesnelle/yarn","repo_kind":"listed","path":"eval/quality.py","file_url":"https://github.com/jquesnelle/yarn/blob/HEAD/eval/quality.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8d8ff31fdde07297"}},{"code_sha256_prefix":"5034871c1d2783cf","entry":"linear_ramp_mask","repo":"jquesnelle/scaled-rope","repo_kind":"listed","path":"scaled_rope/LlamaDynamicPartNTKScaledRotaryEmbedding.py","file_url":"https://github.com/jquesnelle/scaled-rope/blob/HEAD/scaled_rope/LlamaDynamicPartNTKScaledRotaryEmbedding.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5034871c1d2783cf"}},{"code_sha256_prefix":"628d9ed585c36097","entry":"order","repo":"jquesnelle/yarn","repo_kind":"listed","path":"eval/passkey_hard.py","file_url":"https://github.com/jquesnelle/yarn/blob/HEAD/eval/passkey_hard.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"628d9ed585c36097"}},{"code_sha256_prefix":"53b2e52873fb8342","entry":"repeat_kv","repo":"jquesnelle/scaled-rope","repo_kind":"listed","path":"scaled_rope/modeling_llama_together_yarn.py","file_url":"https://github.com/jquesnelle/scaled-rope/blob/HEAD/scaled_rope/modeling_llama_together_yarn.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"53b2e52873fb8342"}},{"code_sha256_prefix":"b99eea6376d1e212","entry":"rotate_half","repo":"jquesnelle/scaled-rope","repo_kind":"listed","path":"scaled_rope/modeling_llama_yarn.py","file_url":"https://github.com/jquesnelle/scaled-rope/blob/HEAD/scaled_rope/modeling_llama_yarn.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b99eea6376d1e212"}},{"code_sha256_prefix":"a7fefb7833cddfa4","entry":"test_model","repo":"jquesnelle/yarn","repo_kind":"listed","path":"eval/passkey.py","file_url":"https://github.com/jquesnelle/yarn/blob/HEAD/eval/passkey.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a7fefb7833cddfa4"}},{"code_sha256_prefix":"9fb7bff934e0d12d","entry":"test_model","repo":"jquesnelle/yarn","repo_kind":"listed","path":"eval/passkey_hard.py","file_url":"https://github.com/jquesnelle/yarn/blob/HEAD/eval/passkey_hard.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9fb7bff934e0d12d"}},{"code_sha256_prefix":"7dc433c456d9abe2","entry":"load_model","repo":"jquesnelle/yarn","repo_kind":"listed","path":"eval/model_loader.py","file_url":"https://github.com/jquesnelle/yarn/blob/HEAD/eval/model_loader.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7dc433c456d9abe2"}},{"code_sha256_prefix":"e20d68d7fd93604b","entry":"main","repo":"jquesnelle/yarn","repo_kind":"listed","path":"tokenization.py","file_url":"https://github.com/jquesnelle/yarn/blob/HEAD/tokenization.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e20d68d7fd93604b"}},{"code_sha256_prefix":"6d86c0fdf41294d9","entry":"main","repo":"jquesnelle/yarn","repo_kind":"listed","path":"truncate.py","file_url":"https://github.com/jquesnelle/yarn/blob/HEAD/truncate.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6d86c0fdf41294d9"}},{"code_sha256_prefix":"51e316fb0b5e2246","entry":"main","repo":"jquesnelle/yarn","repo_kind":"listed","path":"eval/perplexity.py","file_url":"https://github.com/jquesnelle/yarn/blob/HEAD/eval/perplexity.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"51e316fb0b5e2246"}},{"code_sha256_prefix":"ab6fba34dd3815d7","entry":"rmsnorm_func","repo":"jquesnelle/scaled-rope","repo_kind":"listed","path":"scaled_rope/modeling_llama_together_yarn.py","file_url":"https://github.com/jquesnelle/scaled-rope/blob/HEAD/scaled_rope/modeling_llama_together_yarn.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ab6fba34dd3815d7"}},{"code_sha256_prefix":"766ad0e51b7a8f09","entry":"rotate_half","repo":"jquesnelle/scaled-rope","repo_kind":"listed","path":"scaled_rope/FalconDynamicPartNTKScaledRotaryEmbedding.py","file_url":"https://github.com/jquesnelle/scaled-rope/blob/HEAD/scaled_rope/FalconDynamicPartNTKScaledRotaryEmbedding.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"766ad0e51b7a8f09"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}