{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/lamini-lm-a-diverse-herd-of-distilled-models","title":"LaMini-LM: A Diverse Herd of Distilled Models from Large-Scale Instructions","arxiv_id":"2304.14402","date":"2023-04-27","proceeding":null,"authors":["Minghao Wu","Abdul Waheed","Chiyu Zhang","Muhammad Abdul-Mageed","Alham Fikri Aji"],"abstract":"Large language models (LLMs) with instruction fine-tuning demonstrate superior generative capabilities. However, these models are resource-intensive. To alleviate this issue, we explore distilling knowledge from instruction-tuned LLMs into much smaller ones. To this end, we carefully develop a large set of 2.58M instructions based on both existing and newly-generated instructions. In addition to being sizable, we design our instructions to cover a broad set of topics to ensure diversity. Extensive analysis of our instruction dataset confirms its diversity, and we generate responses for these instructions using gpt-3.5-turbo. Leveraging these instructions, we fine-tune a diverse herd of models, collectively referred to as LaMini-LM, which includes models from both the encoder-decoder and decoder-only families, with varying sizes. We evaluate the performance of our models using automatic metrics on 15 different natural language processing (NLP) benchmarks, as well as through human assessment. The results demonstrate that our proposed LaMini-LM models are comparable to competitive baselines, while being much smaller in size.","url_abs":"https://arxiv.org/abs/2304.14402v3","url_pdf":"https://arxiv.org/pdf/2304.14402v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"lamini-lm-a-diverse-herd-of-distilled-models","repo_url":"https://github.com/mbzuai-nlp/lamini-lm","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}}],"tasks":[{"task_slug":"common-sense-reasoning","task_name":"Common Sense Reasoning"},{"task_slug":"coreference-resolution","task_name":"Coreference Resolution"},{"task_slug":"decoder","task_name":"Decoder"},{"task_slug":"diversity","task_name":"Diversity"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"natural-language-inference","task_name":"Natural Language Inference"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"sentence-completion","task_name":"Sentence Completion"},{"task_slug":"word-sense-disambiguation","task_name":"Word Sense Disambiguation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/common-sense-reasoning-on-winogrande","task":"Common Sense Reasoning","dataset":"WinoGrande","model":"FLAN-T5-Large 783M","rank_in_archive_order":51,"of":77,"metrics":{"Accuracy":"59.9"},"uses_additional_data":false},{"leaderboard":"/sota/common-sense-reasoning-on-winogrande","task":"Common Sense Reasoning","dataset":"WinoGrande","model":"GPT-2-XL 1.5B","rank_in_archive_order":56,"of":77,"metrics":{"Accuracy":"58.3"},"uses_additional_data":false},{"leaderboard":"/sota/common-sense-reasoning-on-winogrande","task":"Common Sense Reasoning","dataset":"WinoGrande","model":"LaMini-F-T5 783M","rank_in_archive_order":60,"of":77,"metrics":{"Accuracy":"56"},"uses_additional_data":false},{"leaderboard":"/sota/common-sense-reasoning-on-winogrande","task":"Common Sense Reasoning","dataset":"WinoGrande","model":"LaMini-GPT 1.5B","rank_in_archive_order":61,"of":77,"metrics":{"Accuracy":"56"},"uses_additional_data":false},{"leaderboard":"/sota/common-sense-reasoning-on-winogrande","task":"Common Sense Reasoning","dataset":"WinoGrande","model":"T5-Large 738M","rank_in_archive_order":64,"of":77,"metrics":{"Accuracy":"55.2"},"uses_additional_data":false},{"leaderboard":"/sota/common-sense-reasoning-on-winogrande","task":"Common Sense Reasoning","dataset":"WinoGrande","model":"LaMini-T5 738M","rank_in_archive_order":65,"of":77,"metrics":{"Accuracy":"54.9"},"uses_additional_data":false},{"leaderboard":"/sota/coreference-resolution-on-winograd-schema","task":"Coreference Resolution","dataset":"Winograd Schema Challenge","model":"GPT-2-XL 1.5B","rank_in_archive_order":29,"of":82,"metrics":{"Accuracy":"73.3"},"uses_additional_data":false},{"leaderboard":"/sota/coreference-resolution-on-winograd-schema","task":"Coreference Resolution","dataset":"Winograd Schema Challenge","model":"LaMini-GPT 1.5B","rank_in_archive_order":35,"of":82,"metrics":{"Accuracy":"69.6"},"uses_additional_data":false},{"leaderboard":"/sota/coreference-resolution-on-winograd-schema","task":"Coreference Resolution","dataset":"Winograd Schema Challenge","model":"T5-Large 738M","rank_in_archive_order":40,"of":82,"metrics":{"Accuracy":"66.7"},"uses_additional_data":false},{"leaderboard":"/sota/coreference-resolution-on-winograd-schema","task":"Coreference Resolution","dataset":"Winograd Schema Challenge","model":"LaMini-F-T5 783M","rank_in_archive_order":44,"of":82,"metrics":{"Accuracy":"64.1"},"uses_additional_data":false},{"leaderboard":"/sota/coreference-resolution-on-winograd-schema","task":"Coreference Resolution","dataset":"Winograd Schema Challenge","model":"LaMini-T5 738M","rank_in_archive_order":61,"of":82,"metrics":{"Accuracy":"59"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-multinli","task":"Natural Language Inference","dataset":"MultiNLI","model":"T5-Large 738M","rank_in_archive_order":48,"of":67,"metrics":{"Matched":"72.4","Mismatched":"72"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-multinli","task":"Natural Language Inference","dataset":"MultiNLI","model":"LaMini-GPT 1.5B","rank_in_archive_order":55,"of":67,"metrics":{"Matched":"67.5","Mismatched":"69.3"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-multinli","task":"Natural Language Inference","dataset":"MultiNLI","model":"LaMini-F-T5 783M","rank_in_archive_order":56,"of":67,"metrics":{"Matched":"61.4","Mismatched":"61"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-multinli","task":"Natural Language Inference","dataset":"MultiNLI","model":"LaMini-T5 738M","rank_in_archive_order":57,"of":67,"metrics":{"Matched":"54.7","Mismatched":"55.8"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-multinli","task":"Natural Language Inference","dataset":"MultiNLI","model":"GPT-2-XL 1.5B","rank_in_archive_order":58,"of":67,"metrics":{"Matched":"36.5","Mismatched":"37"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-rte","task":"Natural Language Inference","dataset":"RTE","model":"T5-Large 738M","rank_in_archive_order":20,"of":90,"metrics":{"Accuracy":"87.4%"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-rte","task":"Natural Language Inference","dataset":"RTE","model":"LaMini-GPT 1.5B","rank_in_archive_order":61,"of":90,"metrics":{"Accuracy":"67.9%"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-rte","task":"Natural Language Inference","dataset":"RTE","model":"LaMini-F-T5 783M","rank_in_archive_order":65,"of":90,"metrics":{"Accuracy":"65%"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-rte","task":"Natural Language Inference","dataset":"RTE","model":"LaMini-T5 738M","rank_in_archive_order":81,"of":90,"metrics":{"Accuracy":"57%"},"uses_additional_data":false},{"leaderboard":"/sota/natural-language-inference-on-rte","task":"Natural Language Inference","dataset":"RTE","model":"GPT-2-XL 1.5B","rank_in_archive_order":89,"of":90,"metrics":{"Accuracy":"52.3%"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-openbookqa","task":"Question Answering","dataset":"OpenBookQA","model":"LaMini-GPT 1.5B","rank_in_archive_order":39,"of":45,"metrics":{"Accuracy":"39.8"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-openbookqa","task":"Question Answering","dataset":"OpenBookQA","model":"LaMini-T5 738M","rank_in_archive_order":40,"of":45,"metrics":{"Accuracy":"36"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-openbookqa","task":"Question Answering","dataset":"OpenBookQA","model":"LaMini-F-T5 783M","rank_in_archive_order":41,"of":45,"metrics":{"Accuracy":"34"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-openbookqa","task":"Question Answering","dataset":"OpenBookQA","model":"T5-Large 738M","rank_in_archive_order":42,"of":45,"metrics":{"Accuracy":"32.8"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-openbookqa","task":"Question Answering","dataset":"OpenBookQA","model":"GPT-2-XL 1.5B","rank_in_archive_order":43,"of":45,"metrics":{"Accuracy":"32"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-openbookqa","task":"Question Answering","dataset":"OpenBookQA","model":"FLAN-T5-Large 783M","rank_in_archive_order":44,"of":45,"metrics":{"Accuracy":"31.2"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-piqa","task":"Question Answering","dataset":"PIQA","model":"FLAN-T5-Large 783M","rank_in_archive_order":53,"of":67,"metrics":{"Accuracy":"72.2"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-piqa","task":"Question Answering","dataset":"PIQA","model":"LaMini-GPT 1.5B","rank_in_archive_order":54,"of":67,"metrics":{"Accuracy":"71.3"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-piqa","task":"Question Answering","dataset":"PIQA","model":"LaMini-F-T5 783M","rank_in_archive_order":55,"of":67,"metrics":{"Accuracy":"70.6"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-piqa","task":"Question Answering","dataset":"PIQA","model":"GPT-2-XL 1.5B","rank_in_archive_order":56,"of":67,"metrics":{"Accuracy":"70.5"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-piqa","task":"Question Answering","dataset":"PIQA","model":"LaMini-T5 738M","rank_in_archive_order":60,"of":67,"metrics":{"Accuracy":"67.2"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-piqa","task":"Question Answering","dataset":"PIQA","model":"T5-Large 738M","rank_in_archive_order":65,"of":67,"metrics":{"Accuracy":"55.9"},"uses_additional_data":false},{"leaderboard":"/sota/sentence-completion-on-hellaswag","task":"Sentence Completion","dataset":"HellaSwag","model":"GPT-2-XL 1.5B","rank_in_archive_order":66,"of":89,"metrics":{"Accuracy":"50.9"},"uses_additional_data":false},{"leaderboard":"/sota/sentence-completion-on-hellaswag","task":"Sentence Completion","dataset":"HellaSwag","model":"FLAN-T5-Large 783M","rank_in_archive_order":69,"of":89,"metrics":{"Accuracy":"48.7"},"uses_additional_data":false},{"leaderboard":"/sota/sentence-completion-on-hellaswag","task":"Sentence Completion","dataset":"HellaSwag","model":"LaMini-GPT 1.5B","rank_in_archive_order":70,"of":89,"metrics":{"Accuracy":"48.3"},"uses_additional_data":false},{"leaderboard":"/sota/sentence-completion-on-hellaswag","task":"Sentence Completion","dataset":"HellaSwag","model":"LaMini-F-T5 783M","rank_in_archive_order":72,"of":89,"metrics":{"Accuracy":"43.7"},"uses_additional_data":false},{"leaderboard":"/sota/sentence-completion-on-hellaswag","task":"Sentence Completion","dataset":"HellaSwag","model":"LaMini-T5 738M","rank_in_archive_order":76,"of":89,"metrics":{"Accuracy":"40.6"},"uses_additional_data":false},{"leaderboard":"/sota/sentence-completion-on-hellaswag","task":"Sentence Completion","dataset":"HellaSwag","model":"T5-Large 738M","rank_in_archive_order":78,"of":89,"metrics":{"Accuracy":"38.9"},"uses_additional_data":false},{"leaderboard":"/sota/word-sense-disambiguation-on-words-in-context","task":"Word Sense Disambiguation","dataset":"Words in Context","model":"FLAN-T5-Large 783M","rank_in_archive_order":15,"of":37,"metrics":{"Accuracy":"64.7"},"uses_additional_data":false},{"leaderboard":"/sota/word-sense-disambiguation-on-words-in-context","task":"Word Sense Disambiguation","dataset":"Words in Context","model":"LaMini-F-T5 783M","rank_in_archive_order":16,"of":37,"metrics":{"Accuracy":"63.8"},"uses_additional_data":false},{"leaderboard":"/sota/word-sense-disambiguation-on-words-in-context","task":"Word Sense Disambiguation","dataset":"Words in Context","model":"LaMini-GPT 1.5B","rank_in_archive_order":26,"of":37,"metrics":{"Accuracy":"52.4"},"uses_additional_data":false},{"leaderboard":"/sota/word-sense-disambiguation-on-words-in-context","task":"Word Sense Disambiguation","dataset":"Words in Context","model":"LaMini-T5 738M","rank_in_archive_order":32,"of":37,"metrics":{"Accuracy":"50.5"},"uses_additional_data":false},{"leaderboard":"/sota/word-sense-disambiguation-on-words-in-context","task":"Word Sense Disambiguation","dataset":"Words in Context","model":"GPT-2-XL 1.5B","rank_in_archive_order":34,"of":37,"metrics":{"Accuracy":"49.8"},"uses_additional_data":false}],"syntology":{"syntology_url":"https://syntology.ai/paper/2304.14402","atlas_url":"https://app.syntology.ai/?focus=2304.14402","mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}