{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/language-model-pre-training-with-sparse","title":"Language Model Pre-Training with Sparse Latent Typing","arxiv_id":"2210.12582","date":"2022-10-23","proceeding":null,"authors":["Liliang Ren","Zixuan Zhang","Han Wang","Clare R. Voss","ChengXiang Zhai","Heng Ji"],"abstract":"Modern large-scale Pre-trained Language Models (PLMs) have achieved tremendous success on a wide range of downstream tasks. However, most of the LM pre-training objectives only focus on text reconstruction, but have not sought to learn latent-level interpretable representations of sentences. In this paper, we manage to push the language models to obtain a deeper understanding of sentences by proposing a new pre-training objective, Sparse Latent Typing, which enables the model to sparsely extract sentence-level keywords with diverse latent types. Experimental results show that our model is able to learn interpretable latent type categories in a self-supervised manner without using any external knowledge. Besides, the language model pre-trained with such an objective also significantly improves Information Extraction related downstream tasks in both supervised and few-shot settings. Our code is publicly available at: https://github.com/renll/SparseLT.","url_abs":"https://arxiv.org/abs/2210.12582v2","url_pdf":"https://arxiv.org/pdf/2210.12582v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"language-model-pre-training-with-sparse","repo_url":"https://github.com/renll/sparselt","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"few-shot-ner","task_name":"Few-shot NER"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"sentence","task_name":"Sentence"},{"task_slug":"model","task_name":"model"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/few-shot-ner-on-few-nerd-inter","task":"Few-shot NER","dataset":"Few-NERD (INTER)","model":"BERT-SparseLT + CONTaiNER","rank_in_archive_order":9,"of":13,"metrics":{"10 way 1~2 shot":"52.75","10 way 5~10 shot":"62.43","5 way 1~2 shot":"57.14","5 way 5~10 shot":"66.17"},"uses_additional_data":true},{"leaderboard":"/sota/few-shot-ner-on-few-nerd-intra","task":"Few-shot NER","dataset":"Few-NERD (INTRA)","model":"BERT-SparseLT+CONTainNER","rank_in_archive_order":8,"of":13,"metrics":{"10 way 1~2 shot":"40.48","10 way 5~10 shot":"53.04","5 way 1~2 shot":"47.20","5 way 5~10 shot":"59.67"},"uses_additional_data":true}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2210.12582","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2210.12582"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/renll/sparselt","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_draft_wrong":3,"ran_honours":1,"unverified":8},"by_repo_kind":{"official":{"samples":12,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"d9dcfb510a5e2690","entry":"gelu","repo":"renll/sparselt","repo_kind":"official","path":"model.py","file_url":"https://github.com/renll/sparselt/blob/HEAD/model.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d9dcfb510a5e2690"}},{"code_sha256_prefix":"bd9ad74746bdeace","entry":"log_sum_exp","repo":"renll/sparselt","repo_kind":"official","path":"evaluations/supervised-ie/model.py","file_url":"https://github.com/renll/sparselt/blob/HEAD/evaluations/supervised-ie/model.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bd9ad74746bdeace"}},{"code_sha256_prefix":"f9379a80bc4cfd5a","entry":"sequence_mask","repo":"renll/sparselt","repo_kind":"official","path":"evaluations/supervised-ie/model.py","file_url":"https://github.com/renll/sparselt/blob/HEAD/evaluations/supervised-ie/model.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f9379a80bc4cfd5a"}},{"code_sha256_prefix":"c5fe6d1a240e86e5","entry":"token_lens_to_offsets","repo":"renll/sparselt","repo_kind":"official","path":"evaluations/supervised-ie/model.py","file_url":"https://github.com/renll/sparselt/blob/HEAD/evaluations/supervised-ie/model.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c5fe6d1a240e86e5"}},{"code_sha256_prefix":"fada67a6d1459131","entry":"compute_f1","repo":"renll/sparselt","repo_kind":"official","path":"evaluations/supervised-ie/scorer.py","file_url":"https://github.com/renll/sparselt/blob/HEAD/evaluations/supervised-ie/scorer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fada67a6d1459131"}},{"code_sha256_prefix":"09b5a097f3682dd3","entry":"convert_arguments","repo":"renll/sparselt","repo_kind":"official","path":"evaluations/supervised-ie/scorer.py","file_url":"https://github.com/renll/sparselt/blob/HEAD/evaluations/supervised-ie/scorer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"09b5a097f3682dd3"}},{"code_sha256_prefix":"7977941eee570187","entry":"get_entity_labels","repo":"renll/sparselt","repo_kind":"official","path":"evaluations/supervised-ie/data.py","file_url":"https://github.com/renll/sparselt/blob/HEAD/evaluations/supervised-ie/data.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7977941eee570187"}},{"code_sha256_prefix":"d8fb460267273049","entry":"get_span_mention_text","repo":"renll/sparselt","repo_kind":"official","path":"evaluations/supervised-ie/convert.py","file_url":"https://github.com/renll/sparselt/blob/HEAD/evaluations/supervised-ie/convert.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d8fb460267273049"}},{"code_sha256_prefix":"c92222981e1a0a3f","entry":"get_trigger_labels","repo":"renll/sparselt","repo_kind":"official","path":"evaluations/supervised-ie/data.py","file_url":"https://github.com/renll/sparselt/blob/HEAD/evaluations/supervised-ie/data.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c92222981e1a0a3f"}},{"code_sha256_prefix":"422237a1d98228f2","entry":"load_mapping","repo":"renll/sparselt","repo_kind":"official","path":"evaluations/supervised-ie/convert.py","file_url":"https://github.com/renll/sparselt/blob/HEAD/evaluations/supervised-ie/convert.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"422237a1d98228f2"}},{"code_sha256_prefix":"006b8e682f7dfff0","entry":"remove_overlap_entities","repo":"renll/sparselt","repo_kind":"official","path":"evaluations/supervised-ie/data.py","file_url":"https://github.com/renll/sparselt/blob/HEAD/evaluations/supervised-ie/data.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"006b8e682f7dfff0"}},{"code_sha256_prefix":"ac4edaf53cf349f7","entry":"safe_div","repo":"renll/sparselt","repo_kind":"official","path":"evaluations/supervised-ie/scorer.py","file_url":"https://github.com/renll/sparselt/blob/HEAD/evaluations/supervised-ie/scorer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ac4edaf53cf349f7"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}