{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/task-diversity-shortens-the-icl-plateau","title":"Task Diversity Shortens the ICL Plateau","arxiv_id":"2410.05448","date":"2024-10-07","proceeding":null,"authors":["Jaeyeon Kim","Sehyun Kwon","Joo Young Choi","Jongho Park","Jaewoong Cho","Jason D. Lee","Ernest K. Ryu"],"abstract":"In-context learning (ICL) describes a language model's ability to generate outputs based on a set of input demonstrations and a subsequent query. To understand this remarkable capability, researchers have studied simplified, stylized models. These studies have consistently observed long loss plateaus, during which models exhibit minimal improvement, followed by a sudden, rapid surge of learning. In this work, we reveal that training on multiple diverse ICL tasks simultaneously shortens the loss plateaus, making each task easier to learn. This finding is surprising as it contradicts the natural intuition that the combined complexity of multiple ICL tasks would lengthen the learning process, not shorten it. Our result suggests that the recent success in large-scale training of language models may be attributed not only to the richness of the data at scale but also to the easier optimization (training) induced by the diversity of natural language training data.","url_abs":"https://arxiv.org/abs/2410.05448v1","url_pdf":"https://arxiv.org/pdf/2410.05448v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"task-diversity-shortens-the-icl-plateau","repo_url":"https://github.com/sehyunkwon/task-diversity-icl","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"diversity","task_name":"Diversity"},{"task_slug":"in-context-learning","task_name":"In-Context Learning"}],"methods":[{"method_slug":"set","method_name":"SET"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2410.05448","atlas_url":"https://app.syntology.ai/?focus=2410.05448","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.05448"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sehyunkwon/task-diversity-icl","reach":{"status":"ok"}}],"summary":{"ran":9,"ran_draft_wrong":2,"unverified":1},"by_repo_kind":{"official":{"samples":12,"ran":11,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":12,"samples":[{"code_sha256_prefix":"d7ae3e05768f422d","entry":"accuracy","repo":"sehyunkwon/task-diversity-icl","repo_kind":"official","path":"src/bool_task.py","file_url":"https://github.com/sehyunkwon/task-diversity-icl/blob/HEAD/src/bool_task.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d7ae3e05768f422d"}},{"code_sha256_prefix":"420674d96c9c88ae","entry":"adjust_batch_sizes","repo":"sehyunkwon/task-diversity-icl","repo_kind":"official","path":"task_configs/generate.py","file_url":"https://github.com/sehyunkwon/task-diversity-icl/blob/HEAD/task_configs/generate.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"420674d96c9c88ae"}},{"code_sha256_prefix":"41065e1207ba1b0d","entry":"cross_entropy","repo":"sehyunkwon/task-diversity-icl","repo_kind":"official","path":"src/bool_task.py","file_url":"https://github.com/sehyunkwon/task-diversity-icl/blob/HEAD/src/bool_task.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"41065e1207ba1b0d"}},{"code_sha256_prefix":"797281bf91c00aef","entry":"cross_entropy","repo":"sehyunkwon/task-diversity-icl","repo_kind":"official","path":"src/continuous_task.py","file_url":"https://github.com/sehyunkwon/task-diversity-icl/blob/HEAD/src/continuous_task.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"797281bf91c00aef"}},{"code_sha256_prefix":"1ee7db3dafc63aa0","entry":"generate_idx_list","repo":"sehyunkwon/task-diversity-icl","repo_kind":"official","path":"src/bool_task.py","file_url":"https://github.com/sehyunkwon/task-diversity-icl/blob/HEAD/src/bool_task.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1ee7db3dafc63aa0"}},{"code_sha256_prefix":"71292b149b12d917","entry":"get_optimal_ncl_acc","repo":"sehyunkwon/task-diversity-icl","repo_kind":"official","path":"src/ncl.py","file_url":"https://github.com/sehyunkwon/task-diversity-icl/blob/HEAD/src/ncl.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"71292b149b12d917"}},{"code_sha256_prefix":"0157552fb1f390d1","entry":"get_optimal_ncl_loss","repo":"sehyunkwon/task-diversity-icl","repo_kind":"official","path":"src/ncl.py","file_url":"https://github.com/sehyunkwon/task-diversity-icl/blob/HEAD/src/ncl.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0157552fb1f390d1"}},{"code_sha256_prefix":"fc94f27a0973e093","entry":"mean_squared_error","repo":"sehyunkwon/task-diversity-icl","repo_kind":"official","path":"src/continuous_task.py","file_url":"https://github.com/sehyunkwon/task-diversity-icl/blob/HEAD/src/continuous_task.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fc94f27a0973e093"}},{"code_sha256_prefix":"a81285c7684656ca","entry":"nn_scoring_step","repo":"sehyunkwon/task-diversity-icl","repo_kind":"official","path":"src/attention_analysis.py","file_url":"https://github.com/sehyunkwon/task-diversity-icl/blob/HEAD/src/attention_analysis.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"a81285c7684656ca"}},{"code_sha256_prefix":"211a2ef589b849d4","entry":"prefix_scoring_step","repo":"sehyunkwon/task-diversity-icl","repo_kind":"official","path":"src/attention_analysis.py","file_url":"https://github.com/sehyunkwon/task-diversity-icl/blob/HEAD/src/attention_analysis.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"211a2ef589b849d4"}},{"code_sha256_prefix":"348d63a78eb4cb94","entry":"squared_error","repo":"sehyunkwon/task-diversity-icl","repo_kind":"official","path":"src/continuous_task.py","file_url":"https://github.com/sehyunkwon/task-diversity-icl/blob/HEAD/src/continuous_task.py","link_basis":"plan_row","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"348d63a78eb4cb94"}},{"code_sha256_prefix":"3c24ac252fef0868","entry":"get_final_var","repo":"sehyunkwon/task-diversity-icl","repo_kind":"official","path":"src/curriculum.py","file_url":"https://github.com/sehyunkwon/task-diversity-icl/blob/HEAD/src/curriculum.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3c24ac252fef0868"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}