{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/yarn-efficient-context-window-extension-of","title":"YaRN: Efficient Context Window Extension of Large Language Models","arxiv_id":"2309.00071","date":"2023-08-31","proceeding":null,"authors":["Bowen Peng","Jeffrey Quesnelle","Honglu Fan","Enrico Shippole"],"abstract":"Rotary Position Embeddings (RoPE) have been shown to effectively encode positional information in transformer-based language models. However, these models fail to generalize past the sequence length they were trained on. We present YaRN (Yet another RoPE extensioN method), a compute-efficient method to extend the context window of such models, requiring 10x less tokens and 2.5x less training steps than previous methods. Using YaRN, we show that LLaMA models can effectively utilize and extrapolate to context lengths much longer than their original pre-training would allow, while also surpassing previous the state-of-the-art at context window extension. In addition, we demonstrate that YaRN exhibits the capability to extrapolate beyond the limited context of a fine-tuning dataset. The models fine-tuned using YaRN has been made available and reproduced online up to 128k context length at https://github.com/jquesnelle/yarn","url_abs":"https://arxiv.org/abs/2309.00071v2","url_pdf":"https://arxiv.org/pdf/2309.00071v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"yarn-efficient-context-window-extension-of","repo_url":"https://github.com/jquesnelle/scaled-rope","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"yarn-efficient-context-window-extension-of","repo_url":"https://github.com/jquesnelle/yarn","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"yarn-efficient-context-window-extension-of","repo_url":"https://github.com/nvidia/ruler","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"yarn-efficient-context-window-extension-of","repo_url":"https://github.com/qwenlm/qwen2-vl","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"yarn-efficient-context-window-extension-of","repo_url":"https://github.com/qwenlm/qwen2.5-coder","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"yarn-efficient-context-window-extension-of","repo_url":"https://github.com/qwenlm/qwen2.5-vl","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"yarn-efficient-context-window-extension-of","repo_url":"https://github.com/qwenlm/qwq","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"yarn-efficient-context-window-extension-of","repo_url":"https://github.com/tigerresearch/tigerbot","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"yarn-efficient-context-window-extension-of","repo_url":"https://github.com/tongyi-zhiwen/qwenlong-l1","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"yarn-efficient-context-window-extension-of","repo_url":"https://github.com/ymcui/chinese-llama-alpaca-2","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":null,"task_name":"Position"}],"methods":[{"method_slug":"fail","method_name":"fail"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2309.00071","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2309.00071"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/qwenlm/qwen2.5-vl","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/tongyi-zhiwen/qwenlong-l1","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jquesnelle/yarn","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/nvidia/ruler","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/qwenlm/qwen2-vl","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/tigerresearch/tigerbot","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jquesnelle/scaled-rope","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/qwenlm/qwq","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ymcui/chinese-llama-alpaca-2","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/qwenlm/qwen2.5-coder","reach":{"status":"ok"}}],"summary":{"ran":3,"ran_violates":1,"unverified":16},"by_repo_kind":{"listed":{"samples":20,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"d197da35ee544f75","entry":"conflict_make","repo":"qwenlm/qwq","repo_kind":"listed","path":"eval/eval/ifeval_utils/instructions_registry.py","file_url":"https://github.com/qwenlm/qwq/blob/HEAD/eval/eval/ifeval_utils/instructions_registry.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d197da35ee544f75"}},{"code_sha256_prefix":"cdcc85ca09b00f7e","entry":"count_words","repo":"qwenlm/qwq","repo_kind":"listed","path":"eval/eval/ifeval_utils/instructions_util.py","file_url":"https://github.com/qwenlm/qwq/blob/HEAD/eval/eval/ifeval_utils/instructions_util.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cdcc85ca09b00f7e"}},{"code_sha256_prefix":"d338a8cb44aeb6b4","entry":"is_multi_choice","repo":"qwenlm/qwq","repo_kind":"listed","path":"eval/eval/math_opensource.py","file_url":"https://github.com/qwenlm/qwq/blob/HEAD/eval/eval/math_opensource.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d338a8cb44aeb6b4"}},{"code_sha256_prefix":"34d346853108011e","entry":"split_into_sentences","repo":"qwenlm/qwq","repo_kind":"listed","path":"eval/eval/ifeval_utils/instructions_util.py","file_url":"https://github.com/qwenlm/qwq/blob/HEAD/eval/eval/ifeval_utils/instructions_util.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"34d346853108011e"}},{"code_sha256_prefix":"5faad4dd18dd77ef","entry":"clean_if_name","repo":"qwenlm/qwq","repo_kind":"listed","path":"eval/eval/livecodebench_v5_utils/testing_util.py","file_url":"https://github.com/qwenlm/qwq/blob/HEAD/eval/eval/livecodebench_v5_utils/testing_util.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5faad4dd18dd77ef"}},{"code_sha256_prefix":"9188e3d845f5322a","entry":"compute_metrics_from_results","repo":"qwenlm/qwq","repo_kind":"listed","path":"eval/eval/livecodebench_v5_utils/pass_k_utils.py","file_url":"https://github.com/qwenlm/qwq/blob/HEAD/eval/eval/livecodebench_v5_utils/pass_k_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9188e3d845f5322a"}},{"code_sha256_prefix":"09565dfe50b43516","entry":"count_completed_samples","repo":"qwenlm/qwq","repo_kind":"listed","path":"eval/generate_api_answers/infer_multithread.py","file_url":"https://github.com/qwenlm/qwq/blob/HEAD/eval/generate_api_answers/infer_multithread.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"09565dfe50b43516"}},{"code_sha256_prefix":"f79646ddd0a03162","entry":"count_sentences","repo":"qwenlm/qwq","repo_kind":"listed","path":"eval/eval/ifeval_utils/instructions_util.py","file_url":"https://github.com/qwenlm/qwq/blob/HEAD/eval/eval/ifeval_utils/instructions_util.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f79646ddd0a03162"}},{"code_sha256_prefix":"9df10eb02582b09f","entry":"estimate_pass_at_k","repo":"qwenlm/qwq","repo_kind":"listed","path":"eval/eval/livecodebench_v5_utils/pass_k_utils.py","file_url":"https://github.com/qwenlm/qwq/blob/HEAD/eval/eval/livecodebench_v5_utils/pass_k_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9df10eb02582b09f"}},{"code_sha256_prefix":"406597cd2faf5fef","entry":"extract_instance_results","repo":"qwenlm/qwq","repo_kind":"listed","path":"eval/eval/livecodebench_v5_utils/pass_k_utils.py","file_url":"https://github.com/qwenlm/qwq/blob/HEAD/eval/eval/livecodebench_v5_utils/pass_k_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"406597cd2faf5fef"}},{"code_sha256_prefix":"7b5542cec0c01464","entry":"get_content","repo":"qwenlm/qwq","repo_kind":"listed","path":"eval/generate_api_answers/utils_vllm.py","file_url":"https://github.com/qwenlm/qwq/blob/HEAD/eval/generate_api_answers/utils_vllm.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7b5542cec0c01464"}},{"code_sha256_prefix":"292970496395b6c3","entry":"get_qwen_question_template_answer","repo":"qwenlm/qwq","repo_kind":"listed","path":"eval/eval/livecodebench_v5_utils/process_data.py","file_url":"https://github.com/qwenlm/qwq/blob/HEAD/eval/eval/livecodebench_v5_utils/process_data.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"292970496395b6c3"}},{"code_sha256_prefix":"2e92414197a95a18","entry":"get_qwen_reasoning_question_template_answer","repo":"qwenlm/qwq","repo_kind":"listed","path":"eval/eval/livecodebench_v5_utils/process_data.py","file_url":"https://github.com/qwenlm/qwq/blob/HEAD/eval/eval/livecodebench_v5_utils/process_data.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2e92414197a95a18"}},{"code_sha256_prefix":"aa6fde3b731edf5c","entry":"load_code_generation_dataset","repo":"qwenlm/qwq","repo_kind":"listed","path":"eval/eval/livecodebench_v5_utils/process_data.py","file_url":"https://github.com/qwenlm/qwq/blob/HEAD/eval/eval/livecodebench_v5_utils/process_data.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"aa6fde3b731edf5c"}},{"code_sha256_prefix":"e478b5bb18723734","entry":"make_function","repo":"qwenlm/qwq","repo_kind":"listed","path":"eval/eval/livecodebench_v5_utils/testing_util.py","file_url":"https://github.com/qwenlm/qwq/blob/HEAD/eval/eval/livecodebench_v5_utils/testing_util.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e478b5bb18723734"}},{"code_sha256_prefix":"5ada7674f9cc64ee","entry":"preprocess","repo":"qwenlm/qwq","repo_kind":"listed","path":"eval/eval/livecodebench_v5.py","file_url":"https://github.com/qwenlm/qwq/blob/HEAD/eval/eval/livecodebench_v5.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5ada7674f9cc64ee"}},{"code_sha256_prefix":"1f2c82cb51d70854","entry":"removeprefix","repo":"qwenlm/qwq","repo_kind":"listed","path":"eval/eval/ifeval_utils/instructions.py","file_url":"https://github.com/qwenlm/qwq/blob/HEAD/eval/eval/ifeval_utils/instructions.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1f2c82cb51d70854"}},{"code_sha256_prefix":"c18b33dc2c9d3ac3","entry":"removesuffix","repo":"qwenlm/qwq","repo_kind":"listed","path":"eval/eval/ifeval_utils/instructions.py","file_url":"https://github.com/qwenlm/qwq/blob/HEAD/eval/eval/ifeval_utils/instructions.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c18b33dc2c9d3ac3"}},{"code_sha256_prefix":"f23d0f0bf67cc7cb","entry":"truncatefn","repo":"qwenlm/qwq","repo_kind":"listed","path":"eval/eval/livecodebench_v5_utils/testing_util.py","file_url":"https://github.com/qwenlm/qwq/blob/HEAD/eval/eval/livecodebench_v5_utils/testing_util.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f23d0f0bf67cc7cb"}},{"code_sha256_prefix":"97dd1cf2a21b437c","entry":"work","repo":"qwenlm/qwq","repo_kind":"listed","path":"eval/eval/math_opensource.py","file_url":"https://github.com/qwenlm/qwq/blob/HEAD/eval/eval/math_opensource.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"97dd1cf2a21b437c"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}