{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/micronet-for-efficient-language-modeling","title":"MicroNet for Efficient Language Modeling","arxiv_id":"2005.07877","date":"2020-05-16","proceeding":null,"authors":["Zhongxia Yan","Hanrui Wang","Demi Guo","Song Han"],"abstract":"It is important to design compact language models for efficient deployment. We improve upon recent advances in both the language modeling domain and the model-compression domain to construct parameter and computation efficient language models. We use an efficient transformer-based architecture with adaptive embedding and softmax, differentiable non-parametric cache, Hebbian softmax, knowledge distillation, network pruning, and low-bit quantization. In this paper, we provide the winning solution to the NeurIPS 2019 MicroNet Challenge in the language modeling track. Compared to the baseline language model provided by the MicroNet Challenge, our model is 90 times more parameter-efficient and 36 times more computation-efficient while achieving the required test perplexity of 35 on the Wikitext-103 dataset. We hope that this work will aid future research into efficient language models, and we have released our full source code at https://github.com/mit-han-lab/neurips-micronet.","url_abs":"https://arxiv.org/abs/2005.07877v1","url_pdf":"https://arxiv.org/pdf/2005.07877v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"micronet-for-efficient-language-modeling","repo_url":"https://github.com/mit-han-lab/neurips-micronet","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"knowledge-distillation","task_name":"Knowledge Distillation"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"model-compression","task_name":"Model Compression"},{"task_slug":"network-pruning","task_name":"Network Pruning"},{"task_slug":"quantization","task_name":"Quantization"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2005.07877","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2005.07877"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/mit-han-lab/neurips-micronet","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":5},"by_repo_kind":{"official":{"samples":5,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"ef9ad24f968b2da4","entry":"evaluate","repo":"mit-han-lab/neurips-micronet","repo_kind":"official","path":"data.py","file_url":"https://github.com/mit-han-lab/neurips-micronet/blob/HEAD/data.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ef9ad24f968b2da4"}},{"code_sha256_prefix":"ee47806c7544e978","entry":"format_json","repo":"mit-han-lab/neurips-micronet","repo_kind":"official","path":"u.py","file_url":"https://github.com/mit-han-lab/neurips-micronet/blob/HEAD/u.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ee47806c7544e978"}},{"code_sha256_prefix":"3a5906ac3db6e4b8","entry":"get_encoder","repo":"mit-han-lab/neurips-micronet","repo_kind":"official","path":"u.py","file_url":"https://github.com/mit-han-lab/neurips-micronet/blob/HEAD/u.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3a5906ac3db6e4b8"}},{"code_sha256_prefix":"9fd2543dd0d158f8","entry":"load_json","repo":"mit-han-lab/neurips-micronet","repo_kind":"official","path":"u.py","file_url":"https://github.com/mit-han-lab/neurips-micronet/blob/HEAD/u.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9fd2543dd0d158f8"}},{"code_sha256_prefix":"04ade7e30fa78ea4","entry":"scheduler","repo":"mit-han-lab/neurips-micronet","repo_kind":"official","path":"optim.py","file_url":"https://github.com/mit-han-lab/neurips-micronet/blob/HEAD/optim.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"04ade7e30fa78ea4"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}