{"about":{"non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","site":"https://codewithpapers.app","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page","syntology":{"site":"https://syntology.ai","developers":"https://syntology.ai/developers","mcp":{"server":"https://syntology.ai/mcp","transport":"streamable-http","server_card":"https://syntology.ai/.well-known/mcp/server-card.json","auth":{"type":"trial token, no account","trial_token":"https://syntology.ai/api/oauth/trial/token","method":"POST","docs":"https://syntology.ai/developers"}},"have":"https://syntology.ai/api/graph/have?x=<method, arXiv id or title> (free, answers coverage only)","paper_base":"https://syntology.ai/paper/","atlas_base":"https://app.syntology.ai/?focus="},"machine_readable":[{"url":"https://codewithpapers.app/llms.txt","what":"the machine catalog: every machine-readable file, counted"},{"url":"https://codewithpapers.app/index/manifest.json","what":"paper-to-code index by arXiv id, with Syntology's counts"},{"url":"https://codewithpapers.app/search/manifest.json","what":"site search index (titles, authors) and its files"},{"url":"https://codewithpapers.app/download","what":"bulk files: Syntology's layer, described there"},{"url":"https://codewithpapers.app/build_manifest.json","what":"the build record: inputs, counts, exclusions, probes"}]},"url":"/task/masked-language-modeling/papers/5","list_of":"/task/masked-language-modeling","task":"Masked Language Modeling","archive":{"snapshot":"2025-07-28"},"key_notes":{"n_ran_checked":"legacy name, kept unchanged so existing readers do not break: it counts the samples that ran with no instrument failure (honoured, violated, and ran with no contract checked); it does not mean a contract was checked, and the pages print it as 'K with no instrument failure', not 'K checked'","n_constructed":"a sub-count of the samples that ran, never subtracted from them and never a failure: an executed sample whose run returned an instance of its own class (fixture_out_type equals the entry name): the run built an object and did not compute a result (Syntology's RAN record, counts.constructed)"},"syntology_read_at":"2026-09-28T10:30:06+00:00","order":"archive","order_definition":"repositories listed in the archive (most first), then date (newest first), then slug","page":5,"pages_in_order":5,"rows_per_page":100,"rows":[401,475],"of":475,"counts":{"archive_papers_tagged":475,"with_a_code_link":254,"where_syntology_ran_a_sample":67,"not_listed_spam_title":0,"listed":475,"listed_where_code_ran":67,"where_syntology_ran_a_sample_split":{"with_a_run_with_no_instrument_failure":52,"every_run_a_failure_of_syntologys_instrument":15,"listed_with_a_run_with_no_instrument_failure":52,"listed_every_run_a_failure_of_syntologys_instrument":15,"filter":{"states":["a run with no instrument failure","any run, instrument failures included"],"default":"a run with no instrument failure","note":"on the 'only where code ran' pages the default hides, in the browser, the rows where every run was a failure of Syntology's instrument; the second state shows them again. Rows are hidden, never re-ordered; these twins list every row"}},"definition":"distinct papers the archive tags; 'where Syntology ran a sample' counts papers with at least one harvested sample that ran, which is not a correctness claim"},"first_page":"/task/masked-language-modeling","prev":"/task/masked-language-modeling/papers/4","next":null,"papers":[{"url":null,"slug":"composable-sparse-fine-tuning-for-cross-1","title":"Composable Sparse Fine-Tuning for Cross-Lingual Transfer","date":"2021-11-16","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"contextual-representation-learning-beyond","title":"Contextual Representation Learning beyond Masked Language Modeling","date":"2021-11-16","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"dawson-data-augmentation-using-weak","title":"DAWSON: Data Augmentation using Weak Supervision On Natural Language","date":"2021-11-16","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"ds-tod-efficient-domain-specialization-for-1","title":"DS-TOD: Efficient Domain Specialization for Task-Oriented Dialog","date":"2021-11-16","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"generative-prompt-tuning-for-relation","title":"Generative Prompt Tuning for Relation Classification","date":"2021-11-16","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"how-does-the-pre-training-objective-affect","title":"How does the pre-training objective affect what large language models learn about linguistic properties?","date":"2021-11-16","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"is-whole-word-masking-always-better-for","title":"\"Is Whole Word Masking Always Better for Chinese BERT?\": Probing on Chinese Grammatical Error Correction","date":"2021-11-16","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"knowledgeable-prompt-tuning-incorporating-1","title":"Knowledgeable Prompt-tuning: Incorporating Knowledge into Prompt Verbalizer for Text Classification","date":"2021-11-16","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"phrase-aware-unsupervised-constituency","title":"Phrase-aware Unsupervised Constituency Parsing","date":"2021-11-16","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"predicting-attention-sparsity-in-transformers-1","title":"Predicting Attention Sparsity in Transformers","date":"2021-11-16","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"probing-berts-priors-with-serial-reproduction","title":"Probing BERT’s priors with serial reproduction chains","date":"2021-11-16","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"prompt-learning-for-fine-grained-entity-1","title":"Prompt-Learning for Fine-Grained Entity Typing","date":"2021-11-16","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"taco-pre-training-of-deep-transformers-with","title":"TACO: Pre-training of Deep Transformers with Attention Convolution using Disentangled Positional Representation","date":"2021-11-16","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"temporal-language-modeling-for-short-text","title":"Temporal Language Modeling for Short Text Document Classification with Transformers","date":"2021-11-16","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"towards-unified-prompt-tuning-for-few-shot","title":"Towards Unified Prompt Tuning for Few-shot Learning","date":"2021-11-16","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"unsupervised-dependency-graph-network","title":"Unsupervised Dependency Graph Network","date":"2021-11-16","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"joint-unsupervised-and-supervised-training","title":"Joint Unsupervised and Supervised Training for Multilingual ASR","date":"2021-11-15","arxiv_id":"2111.08137","repositories_listed":0,"syntology":null},{"url":null,"slug":"modeling-mathematical-notation-semantics-in","title":"Modeling Mathematical Notation Semantics in Academic Papers","date":"2021-11-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"nict-kyoto-submission-for-the-wmt21-quality","title":"NICT Kyoto Submission for the WMT’21 Quality Estimation Task: Multimetric Multilingual Pretraining for Critical Error Detection","date":"2021-11-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"maximizing-efficiency-of-language-model-pre","title":"Maximizing Efficiency of Language Model Pre-training for Learning Representation","date":"2021-10-13","arxiv_id":"2110.06620","repositories_listed":0,"syntology":null},{"url":null,"slug":"multi-modal-pre-training-for-automated-speech","title":"Multi-Modal Pre-Training for Automated Speech Recognition","date":"2021-10-12","arxiv_id":"2110.09890","repositories_listed":0,"syntology":null},{"url":null,"slug":"image-bert-pre-training-with-online-tokenizer","title":"Image BERT Pre-training with Online Tokenizer","date":"2021-09-29","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"mlim-vision-and-language-model-pre-training","title":"MLIM: Vision-and-Language Model Pre-training with Masked Language and Image Modeling","date":"2021-09-24","arxiv_id":"2109.12178","repositories_listed":0,"syntology":null},{"url":null,"slug":"predicting-attention-sparsity-in-transformers","title":"Predicting Attention Sparsity in Transformers","date":"2021-09-24","arxiv_id":"2109.12188","repositories_listed":0,"syntology":null},{"url":null,"slug":"domain-specific-japanese-electra-model-using","title":"Domain-Specific Japanese ELECTRA Model Using a Small Corpus","date":"2021-09-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"split-and-rephrase-in-a-cross-lingual-manner","title":"Split-and-Rephrase in a Cross-Lingual Manner: A Complete Pipeline","date":"2021-09-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"prompt-learning-for-fine-grained-entity","title":"Prompt-Learning for Fine-Grained Entity Typing","date":"2021-08-24","arxiv_id":"2108.10604","repositories_listed":0,"syntology":null},{"url":null,"slug":"noobs-at-semeval-2021-task-4-masked-language","title":"Noobs at Semeval-2021 Task 4: Masked Language Modeling for abstract answer prediction","date":"2021-08-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"learning-to-sample-replacements-for-electra","title":"Learning to Sample Replacements for ELECTRA Pre-Training","date":"2021-06-25","arxiv_id":"2106.13715","repositories_listed":0,"syntology":null},{"url":null,"slug":"winner-team-mia-at-textvqa-challenge-2021","title":"Winner Team Mia at TextVQA Challenge 2021: Vision-and-Language Representation Learning with Pre-trained Sequence-to-Sequence Model","date":"2021-06-24","arxiv_id":"2106.15332","repositories_listed":0,"syntology":null},{"url":null,"slug":"mst-masked-self-supervised-transformer-for","title":"MST: Masked Self-Supervised Transformer for Visual Representation","date":"2021-06-10","arxiv_id":"2106.05656","repositories_listed":0,"syntology":null},{"url":null,"slug":"bi-granularity-contrastive-learning-for-post","title":"Bi-Granularity Contrastive Learning for Post-Training in Few-Shot Scene","date":"2021-06-04","arxiv_id":"2106.02327","repositories_listed":0,"syntology":null},{"url":null,"slug":"exposing-the-implicit-energy-networks-behind","title":"Exposing the Implicit Energy Networks behind Masked Language Models via Metropolis--Hastings","date":"2021-06-04","arxiv_id":"2106.02736","repositories_listed":0,"syntology":null},{"url":null,"slug":"mg-bert-multi-graph-augmented-bert-for-masked","title":"MG-BERT: Multi-Graph Augmented BERT for Masked Language Modeling","date":"2021-06-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"script-self-critic-pretraining-of","title":"SCRIPT: Self-Critic PreTraining of Transformers","date":"2021-06-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"target-aware-data-augmentation-for-stance","title":"Target-Aware Data Augmentation for Stance Detection","date":"2021-06-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"larger-scale-transformers-for-multilingual","title":"Larger-Scale Transformers for Multilingual Masked Language Modeling","date":"2021-05-02","arxiv_id":"2105.00572","repositories_listed":0,"syntology":null},{"url":null,"slug":"understanding-chinese-video-and-language-via","title":"Understanding Chinese Video and Language via Contrastive Multimodal Pre-Training","date":"2021-04-19","arxiv_id":"2104.09411","repositories_listed":0,"syntology":null},{"url":null,"slug":"on-the-influence-of-masking-policies-in","title":"On the Influence of Masking Policies in Intermediate Pre-training","date":"2021-04-18","arxiv_id":"2104.08840","repositories_listed":0,"syntology":null},{"url":null,"slug":"masked-language-modeling-and-the","title":"Masked Language Modeling and the Distributional Hypothesis: Order Word Matters Pre-training for Little","date":"2021-04-14","arxiv_id":"2104.06644","repositories_listed":0,"syntology":null},{"url":null,"slug":"pseudo-label-guided-unsupervised-domain","title":"Pseudo-Label Guided Unsupervised Domain Adaptation of Contextual Embeddings","date":"2021-04-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"uc2-universal-cross-lingual-cross-modal","title":"UC2: Universal Cross-lingual Cross-modal Vision-and-Language Pre-training","date":"2021-04-01","arxiv_id":"2104.00332","repositories_listed":0,"syntology":null},{"url":null,"slug":"self-supervised-image-text-pre-training-with","title":"Self-supervised Image-text Pre-training With Mixed Data In Chest X-rays","date":"2021-03-30","arxiv_id":"2103.16022","repositories_listed":0,"syntology":null},{"url":null,"slug":"variable-name-recovery-in-decompiled-binary","title":"Variable Name Recovery in Decompiled Binary Code using Constrained Masked Language Modeling","date":"2021-03-23","arxiv_id":"2103.12801","repositories_listed":0,"syntology":null},{"url":null,"slug":"a-primer-on-contrastive-pretraining-in","title":"A Primer on Contrastive Pretraining in Language Processing: Methods, Lessons Learned and Perspectives","date":"2021-02-25","arxiv_id":"2102.12982","repositories_listed":0,"syntology":null},{"url":null,"slug":"bilingual-language-modeling-a-transfer","title":"Bilingual Language Modeling, A transfer learning technique for Roman Urdu","date":"2021-02-22","arxiv_id":"2102.10958","repositories_listed":0,"syntology":null},{"url":null,"slug":"mermaid-metaphor-generation-with-symbolism-1","title":"MERMAID: Metaphor Generation with Symbolism and Discriminative Decoding","date":"2021-01-23","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"universal-sentence-representations-learning","title":"Universal Sentence Representations Learning with Conditional Masked Language Model","date":"2021-01-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"universal-sentence-representation-learning","title":"Universal Sentence Representation Learning with Conditional Masked Language Model","date":"2020-12-28","arxiv_id":"2012.14388","repositories_listed":0,"syntology":null},{"url":null,"slug":"profile-prediction-an-alignment-based-pre","title":"Profile Prediction: An Alignment-Based Pre-Training Task for Protein Sequence Models","date":"2020-12-01","arxiv_id":"2012.00195","repositories_listed":0,"syntology":null},{"url":null,"slug":"self-supervised-relationship-probing","title":"Self-Supervised Relationship Probing","date":"2020-12-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":"/paper/xhate-999-analyzing-and-detecting-abusive","slug":"xhate-999-analyzing-and-detecting-abusive","title":"XHate-999: Analyzing and Detecting Abusive Language Across Domains and Languages","date":"2020-12-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"self-supervised-learning-with-cross-modal","title":"Self-Supervised learning with cross-modal transformers for emotion recognition","date":"2020-11-20","arxiv_id":"2011.10652","repositories_listed":0,"syntology":null},{"url":null,"slug":"a-hierarchical-multi-modal-encoder-for-moment","title":"A Hierarchical Multi-Modal Encoder for Moment Localization in Video Corpus","date":"2020-11-18","arxiv_id":"2011.09046","repositories_listed":0,"syntology":null},{"url":null,"slug":"postech-etris-submission-to-the-wmt2020-ape","title":"POSTECH-ETRI’s Submission to the WMT2020 APE Shared Task: Automatic Post-Editing with Cross-lingual Language Model","date":"2020-11-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"effective-decoder-masking-for-transformer","title":"Effective Decoder Masking for Transformer Based End-to-End Speech Recognition","date":"2020-10-27","arxiv_id":"2010.14764","repositories_listed":0,"syntology":null},{"url":null,"slug":"dict-mlm-improved-multilingual-pre-training","title":"DICT-MLM: Improved Multilingual Pre-Training using Bilingual Dictionaries","date":"2020-10-23","arxiv_id":"2010.12566","repositories_listed":0,"syntology":null},{"url":null,"slug":"st-bert-cross-modal-language-model-pre","title":"ST-BERT: Cross-modal Language Model Pre-training For End-to-end Spoken Language Understanding","date":"2020-10-23","arxiv_id":"2010.12283","repositories_listed":0,"syntology":null},{"url":null,"slug":"corruption-is-not-all-bad-incorporating","title":"Corruption Is Not All Bad: Incorporating Discourse Structure into Pre-training via Corruption for Essay Scoring","date":"2020-10-13","arxiv_id":"2010.06137","repositories_listed":0,"syntology":null},{"url":null,"slug":"veco-variable-encoder-decoder-pre-training","title":"VECO: Variable Encoder-decoder Pre-training for Cross-lingual Understanding and Generation","date":"2020-09-28","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"learning-visual-representations-with-caption","title":"Learning Visual Representations with Caption Annotations","date":"2020-08-04","arxiv_id":"2008.01392","repositories_listed":0,"syntology":null},{"url":null,"slug":"tensorcoder-dimension-wise-attention-via","title":"TensorCoder: Dimension-Wise Attention via Tensor Representation for Natural Language Modeling","date":"2020-07-28","arxiv_id":"2008.01547","repositories_listed":0,"syntology":null},{"url":null,"slug":"position-masking-for-language-models","title":"Position Masking for Language Models","date":"2020-06-02","arxiv_id":"2006.05676","repositories_listed":0,"syntology":null},{"url":null,"slug":"segatron-segment-aware-transformer-for","title":"Segatron: Segment-aware Transformer for Language Modeling and Understanding","date":"2020-06-02","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"pre-training-is-almost-all-you-need-an","title":"Pre-training Is (Almost) All You Need: An Application to Commonsense Reasoning","date":"2020-04-29","arxiv_id":"2004.14074","repositories_listed":0,"syntology":null},{"url":null,"slug":"uhh-lt-lt2-at-semeval-2020-task-12-fine","title":"UHH-LT at SemEval-2020 Task 12: Fine-Tuning of Pre-Trained Transformer Networks for Offensive Language Detection","date":"2020-04-23","arxiv_id":"2004.11493","repositories_listed":0,"syntology":null},{"url":"/paper/xgpt-cross-modal-generative-pre-training-for","slug":"xgpt-cross-modal-generative-pre-training-for","title":"XGPT: Cross-modal Generative Pre-Training for Image Captioning","date":"2020-03-03","arxiv_id":"2003.01473","repositories_listed":0,"syntology":null},{"url":"/paper/imagebert-cross-modal-pre-training-with-large","slug":"imagebert-cross-modal-pre-training-with-large","title":"ImageBERT: Cross-modal Pre-training with Large-scale Weak-supervised Image-Text Data","date":"2020-01-22","arxiv_id":"2001.07966","repositories_listed":0,"syntology":null},{"url":null,"slug":"retrieval-augmented-language-model-pre","title":"Retrieval Augmented Language Model Pre-Training","date":"2020-01-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"emerging-cross-lingual-structure-in","title":"Emerging Cross-lingual Structure in Pretrained Language Models","date":"2019-11-04","arxiv_id":"1911.01464","repositories_listed":0,"syntology":null},{"url":null,"slug":"uniter-learning-universal-image-text","title":"UNITER: Learning UNiversal Image-TExt Representations","date":"2019-09-25","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"the-bottom-up-evolution-of-representations-in","title":"The Bottom-up Evolution of Representations in the Transformer: A Study with Machine Translation and Language Modeling Objectives","date":"2019-09-03","arxiv_id":"1909.01380","repositories_listed":0,"syntology":null},{"url":"/paper/unicoder-vl-a-universal-encoder-for-vision","slug":"unicoder-vl-a-universal-encoder-for-vision","title":"Unicoder-VL: A Universal Encoder for Vision and Language by Cross-modal Pre-training","date":"2019-08-16","arxiv_id":"1908.06066","repositories_listed":0,"syntology":null},{"url":null,"slug":"bert-masked-language-modeling-for-co","title":"BERT Masked Language Modeling for Co-reference Resolution","date":"2019-08-01","arxiv_id":null,"repositories_listed":0,"syntology":null},{"url":null,"slug":"190600238","title":"Adversarial Generation and Encoding of Nested Texts","date":"2019-06-01","arxiv_id":"1906.00238","repositories_listed":0,"syntology":null}],"record_sha256":"e1ed8d11ea4573120ce4f1f307c0e117cfcdb2fa60b4a20b50fa5455f2090e7a","record_changed_at":"2026-09-28","record_changed_at_basis":"first_hashed"}