{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/learning-sparse-prototypes-for-text","title":"Learning Sparse Prototypes for Text Generation","arxiv_id":"2006.16336","date":"2020-06-29","proceeding":"NeurIPS 2020 12","authors":["Junxian He","Taylor Berg-Kirkpatrick","Graham Neubig"],"abstract":"Prototype-driven text generation uses non-parametric models that first choose from a library of sentence \"prototypes\" and then modify the prototype to generate the output text. While effective, these methods are inefficient at test time as a result of needing to store and index the entire training corpus. Further, existing methods often require heuristics to identify which prototypes to reference at training time. In this paper, we propose a novel generative model that automatically learns a sparse prototype support set that, nonetheless, achieves strong language modeling performance. This is achieved by (1) imposing a sparsity-inducing prior on the prototype selection distribution, and (2) utilizing amortized variational inference to learn a prototype retrieval function. In experiments, our model outperforms previous prototype-driven language models while achieving up to a 1000x memory reduction, as well as a 1000x speed-up at test time. More interestingly, we show that the learned prototypes are able to capture semantics and syntax at different granularity as we vary the sparsity of prototype selection, and that certain sentence attributes can be controlled by specifying the prototype for generation.","url_abs":"https://arxiv.org/abs/2006.16336v2","url_pdf":"https://arxiv.org/pdf/2006.16336v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"learning-sparse-prototypes-for-text","repo_url":"https://github.com/jxhe/sparse-text-prototype","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"prototype-selection","task_name":"Prototype Selection"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"sentence","task_name":"Sentence"},{"task_slug":"text-generation","task_name":"Text Generation"},{"task_slug":"variational-inference","task_name":"Variational Inference"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2006.16336","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2006.16336"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/jxhe/sparse-text-prototype","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":1,"unverified":8},"by_repo_kind":{"official":{"samples":9,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"992489a59b9e08d0","entry":"apply_to_sample","repo":"jxhe/sparse-text-prototype","repo_kind":"official","path":"sparse_prototype/sp_criterion.py","file_url":"https://github.com/jxhe/sparse-text-prototype/blob/HEAD/sparse_prototype/sp_criterion.py","link_basis":"plan_row","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"992489a59b9e08d0"}},{"code_sha256_prefix":"93f6e2cb46ccec10","entry":"Embedding","repo":"jxhe/sparse-text-prototype","repo_kind":"official","path":"sparse_prototype/retriever/cnn_text.py","file_url":"https://github.com/jxhe/sparse-text-prototype/blob/HEAD/sparse_prototype/retriever/cnn_text.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"93f6e2cb46ccec10"}},{"code_sha256_prefix":"718c5ada347d1b35","entry":"LSTM","repo":"jxhe/sparse-text-prototype","repo_kind":"official","path":"sparse_prototype/sp_model.py","file_url":"https://github.com/jxhe/sparse-text-prototype/blob/HEAD/sparse_prototype/sp_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"718c5ada347d1b35"}},{"code_sha256_prefix":"7339bbc858af2c47","entry":"LSTMCell","repo":"jxhe/sparse-text-prototype","repo_kind":"official","path":"sparse_prototype/sp_model.py","file_url":"https://github.com/jxhe/sparse-text-prototype/blob/HEAD/sparse_prototype/sp_model.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7339bbc858af2c47"}},{"code_sha256_prefix":"135b3dc835ffe6ad","entry":"get_confirm_token","repo":"jxhe/sparse-text-prototype","repo_kind":"official","path":"sparse_prototype/prepare_data.py","file_url":"https://github.com/jxhe/sparse-text-prototype/blob/HEAD/sparse_prototype/prepare_data.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"135b3dc835ffe6ad"}},{"code_sha256_prefix":"2c3cce785199ac95","entry":"label_smoothed_nll_loss","repo":"jxhe/sparse-text-prototype","repo_kind":"official","path":"sparse_prototype/guu_criterion.py","file_url":"https://github.com/jxhe/sparse-text-prototype/blob/HEAD/sparse_prototype/guu_criterion.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2c3cce785199ac95"}},{"code_sha256_prefix":"33456d398c6c40ab","entry":"label_smoothed_nll_loss","repo":"jxhe/sparse-text-prototype","repo_kind":"official","path":"sparse_prototype/lm_criterion.py","file_url":"https://github.com/jxhe/sparse-text-prototype/blob/HEAD/sparse_prototype/lm_criterion.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"33456d398c6c40ab"}},{"code_sha256_prefix":"31234bde077b0786","entry":"move_to_cpu","repo":"jxhe/sparse-text-prototype","repo_kind":"official","path":"sparse_prototype/sp_criterion.py","file_url":"https://github.com/jxhe/sparse-text-prototype/blob/HEAD/sparse_prototype/sp_criterion.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"31234bde077b0786"}},{"code_sha256_prefix":"dd2b25b511012165","entry":"parse_lambda_config","repo":"jxhe/sparse-text-prototype","repo_kind":"official","path":"sparse_prototype/sp_task.py","file_url":"https://github.com/jxhe/sparse-text-prototype/blob/HEAD/sparse_prototype/sp_task.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dd2b25b511012165"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}