{"url":"/method/mirror-bert","slug":"mirror-bert","name":"Mirror-BERT","full_name":"Mirror-BERT","full_name_withheld":false,"description_markdown":"Mirror-BERT converts pretrained language models into effective universal text encoders without any supervision, in 20-30 seconds. It is an extremely simple, fast, and effective contrastive learning technique. It relies on fully identical *or* slightly modified string pairs as positive (i.e., synonymous) fine-tuning examples, and aims to maximise their similarity during identity fine-tuning.","description_state":"present","introduced_year":null,"introduced_by":{"title":"Fast, Effective, and Self-Supervised: Transforming Masked Language Models into Universal Lexical and Sentence Encoders","paper":"/paper/fast-effective-and-self-supervised","first_author":"Fangyu Liu","n_authors":4,"url_abs":null,"archive_paper_url":"https://paperswithcode.com/paper/fast-effective-and-self-supervised"},"source":{"url":"https://arxiv.org/abs/2104.08027v2","title":"Fast, Effective, and Self-Supervised: Transforming Masked Language Models into Universal Lexical and Sentence Encoders","url_on_a_paper_host":true},"code_snippet_url":"https://github.com/cambridgeltl/mirror-bert","code_snippet_url_on_a_code_host":true,"categories":[{"area":"Natural Language Processing","area_id":"natural-language-processing","collection":"Static Word Embeddings","url":"/methods/category/static-word-embeddings","pwc_aliases":[]},{"area":"Natural Language Processing","area_id":"natural-language-processing","collection":"Contextualized Word Embeddings","url":"/methods/category/contextualized-word-embeddings","pwc_aliases":[]},{"area":"Natural Language Processing","area_id":"natural-language-processing","collection":"Word Embeddings","url":"/methods/category/word-embeddings","pwc_aliases":[]},{"area":"Natural Language Processing","area_id":"natural-language-processing","collection":"Sentence Embeddings","url":"/methods/category/sentence-embeddings","pwc_aliases":[]},{"area":"General","area_id":"general","collection":"Self-Supervised Learning","url":"/methods/category/self-supervised-learning","pwc_aliases":[]}],"n_papers_tagged":6,"archive_num_papers":6,"papers_newest_first":[{"paper":"/paper/tacl-improving-bert-pre-training-with-token","title":"TaCL: Improving BERT Pre-training with Token-aware Contrastive Learning","date":"2021-11-07","arxiv_id":"2111.04198","n_code_links":2,"syntology":null},{"paper":"/paper/prix-lm-pretraining-for-multilingual","title":"Prix-LM: Pretraining for Multilingual Knowledge Base Construction","date":"2021-10-16","arxiv_id":"2110.08443","n_code_links":1,"syntology":null},{"paper":"/paper/rewire-then-probe-a-contrastive-recipe-for","title":"Rewire-then-Probe: A Contrastive Recipe for Probing Biomedical Knowledge of Pre-trained Language Models","date":"2021-10-15","arxiv_id":"2110.08173","n_code_links":1,"syntology":null},{"paper":"/paper/trans-encoder-unsupervised-sentence-pair","title":"Trans-Encoder: Unsupervised sentence-pair modelling through self- and mutual-distillations","date":"2021-09-27","arxiv_id":"2109.13059","n_code_links":1,"syntology":null},{"paper":"/paper/mirrorwic-on-eliciting-word-in-context","title":"MirrorWiC: On Eliciting Word-in-Context Representations from Pretrained Language Models","date":"2021-09-19","arxiv_id":"2109.09237","n_code_links":1,"syntology":null},{"paper":"/paper/fast-effective-and-self-supervised","title":"Fast, Effective, and Self-Supervised: Transforming Masked Language Models into Universal Lexical and Sentence Encoders","date":"2021-04-16","arxiv_id":"2104.08027","n_code_links":1,"syntology":{"ran":0,"of":1,"unverified":1,"pointer_only":0}}],"papers_shown":6,"tasks":[{"task":"/task/contrastive-learning","name":"Contrastive Learning","papers":4},{"task":"/task/sentence","name":"Sentence","papers":3},{"task":"/task/entity-linking","name":"Entity Linking","papers":2},{"task":"/task/language-modelling","name":"Language Modelling","papers":2},{"task":"/task/semantic-textual-similarity","name":"Semantic Textual Similarity","papers":2},{"task":"/task/sentence-similarity","name":"Sentence Similarity","papers":2},{"task":"/task/bilingual-lexicon-induction","name":"Bilingual Lexicon Induction","papers":1},{"task":null,"name":"Causal Language Modeling","papers":1},{"task":"/task/contextualised-word-representations","name":"Contextualised Word Representations","papers":1},{"task":"/task/cross-lingual-entity-linking","name":"Cross-Lingual Entity Linking","papers":1},{"task":"/task/cross-lingual-semantic-textual-similarity","name":"Cross-Lingual Semantic Textual Similarity","papers":1},{"task":"/task/knowledge-base-construction","name":"Knowledge Base Construction","papers":1},{"task":"/task/knowledge-probing","name":"Knowledge Probing","papers":1},{"task":"/task/language-modeling","name":"Language Modeling","papers":1},{"task":"/task/link-prediction","name":"Link Prediction","papers":1},{"task":"/task/natural-language-understanding","name":"Natural Language Understanding","papers":1},{"task":"/task/paraphrase-identification","name":"Paraphrase Identification","papers":1},{"task":"/task/sts","name":"STS","papers":1},{"task":"/task/semantic-similarity","name":"Semantic Similarity","papers":1},{"task":"/task/transfer-learning","name":"Transfer Learning","papers":1}],"tasks_shown":20,"n_tasks":22,"usage_by_year":[{"year":"2021","papers":6}],"row_source":"methods_table","archive":{"source":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","archive_url":"https://paperswithcode.com/method/mirror-bert"},"syntology_read_at":"2026-09-24T18:15:14+00:00"}