{"url":"/dataset/cola","name":"CoLA","full_name":"Corpus of Linguistic Acceptability","description_markdown":"The **Corpus of Linguistic Acceptability** (**CoLA**) consists of 10657 sentences from 23 linguistics publications, expertly annotated for acceptability (grammaticality) by their original authors. The public version contains 9594 sentences belonging to training and development sets, and excludes 1063 sentences belonging to a held out test set.\r\n\r\nSource: [https://nyu-mll.github.io/CoLA/](https://nyu-mll.github.io/CoLA/)\r\nImage Source: [https://arxiv.org/pdf/1805.12471.pdf](https://arxiv.org/pdf/1805.12471.pdf)","description_withheld":null,"homepage":"https://nyu-mll.github.io/CoLA/","introduced_date":"2018-01-01","introduced_date_note":null,"introduced_by":{"paper":"/paper/neural-network-acceptability-judgments","title":"Neural Network Acceptability Judgments","first_author":"Alex Warstadt","url":null},"license":{"name":"Custom","url":"https://nyu-mll.github.io/CoLA/"},"modalities":[{"name":"Texts","url":"/datasets/modality/texts"}],"tasks":[{"name":"Text Generation","url":"/task/text-generation","datasets_with_task":"/datasets/task/text-generation"},{"name":"Stochastic Optimization","url":"/task/stochastic-optimization","datasets_with_task":"/datasets/task/stochastic-optimization"},{"name":"Linguistic Acceptability","url":"/task/linguistic-acceptability","datasets_with_task":"/datasets/task/linguistic-acceptability"}],"languages":[{"name":"Spanish","url":"/datasets/language/spanish"}],"variants":["CoLA","CoLA Dev"],"data_loaders":[],"num_papers_in_archive":710,"source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28"},"benchmarks":[{"leaderboard":"/sota/linguistic-acceptability-on-cola","task":"Linguistic Acceptability","dataset_variant":"CoLA","rows":43,"metrics":["Accuracy","MCC"],"first_row_in_archive_order":{"model":"En-BERT + TDA + PCA","paper":"/paper/acceptability-judgements-via-examining-the","metrics":{"Accuracy":"88.6%"},"code_links":[{"title":"danchern97/tda4la","url":"https://github.com/danchern97/tda4la"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/linguistic-acceptability-on-cola-dev","task":"Linguistic Acceptability","dataset_variant":"CoLA Dev","rows":6,"metrics":["Accuracy","MCC"],"first_row_in_archive_order":{"model":"En-BERT + TDA","paper":"/paper/acceptability-judgements-via-examining-the","metrics":{"Accuracy":"88.6","MCC":"0.725"},"code_links":[{"title":"danchern97/tda4la","url":"https://github.com/danchern97/tda4la"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/stochastic-optimization-on-cola","task":"Stochastic Optimization","dataset_variant":"CoLA","rows":1,"metrics":["Accuracy (max)","Accuracy (mean)"],"first_row_in_archive_order":{"model":"Bert","paper":"/paper/mixing-adam-and-sgd-a-combined-optimization","metrics":{"Accuracy (max)":"86.34","Accuracy (mean)":"87.66"},"code_links":[{"title":"nicolalandro/multi_optimizer","url":"https://gitlab.com/nicolalandro/multi_optimizer"}]},"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"},{"leaderboard":"/sota/text-generation-on-cola","task":"Text Generation","dataset_variant":"CoLA","rows":0,"metrics":["acc"],"first_row_in_archive_order":null,"note":"rows are the archive's own order at snapshot; nothing here re-ranks them"}],"papers_with_a_benchmark_row":[{"paper":"/paper/not-all-layers-are-equally-as-important-every","title":"Not all layers are equally as important: Every Layer Counts BERT","date":"2023-11-03","rows_on_this_dataset":4,"code_links":0,"syntology":null},{"paper":"/paper/lm-cppf-paraphrasing-guided-data-augmentation","title":"LM-CPPF: Paraphrasing-Guided Data Augmentation for Contrastive Prompt-Based Few-Shot Fine-Tuning","date":"2023-05-29","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/can-bert-eat-rucola-topological-data-analysis","title":"Can BERT eat RuCoLA? Topological Data Analysis to Explain","date":"2023-04-04","rows_on_this_dataset":2,"code_links":2,"syntology":null},{"paper":"/paper/texttt-tasksource-structured-dataset","title":"tasksource: A Dataset Harmonization Framework for Streamlined NLP Multi-Task Learning and Evaluation","date":"2023-01-14","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":4,"samples_ran":3,"samples_unverified":1,"pointer_only_for_licence":4,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/rucola-russian-corpus-of-linguistic","title":"RuCoLA: Russian Corpus of Linguistic Acceptability","date":"2022-10-23","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/llm-int8-8-bit-matrix-multiplication-for","title":"LLM.int8(): 8-bit Matrix Multiplication for Transformers at Scale","date":"2022-08-15","rows_on_this_dataset":1,"code_links":4,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":5,"samples_ran":2,"samples_unverified":3,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/acceptability-judgements-via-examining-the","title":"Acceptability Judgements via Examining the Topology of Attention Maps","date":"2022-05-19","rows_on_this_dataset":5,"code_links":1,"syntology":null},{"paper":"/paper/data2vec-a-general-framework-for-self-1","title":"data2vec: A General Framework for Self-supervised Learning in Speech, Vision and Language","date":"2022-02-07","rows_on_this_dataset":1,"code_links":12,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":6,"samples_ran":0,"samples_unverified":6,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/charformer-fast-character-transformers-via","title":"Charformer: Fast Character Transformers via Gradient-based Subword Tokenization","date":"2021-06-23","rows_on_this_dataset":1,"code_links":2,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":10,"samples_ran":7,"samples_unverified":3,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/fnet-mixing-tokens-with-fourier-transforms","title":"FNet: Mixing Tokens with Fourier Transforms","date":"2021-05-09","rows_on_this_dataset":1,"code_links":12,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":2,"samples_ran":2,"samples_unverified":0,"pointer_only_for_licence":1,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/entailment-as-few-shot-learner","title":"Entailment as Few-Shot Learner","date":"2021-04-29","rows_on_this_dataset":1,"code_links":3,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":3,"samples_ran":1,"samples_unverified":2,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/how-to-train-bert-with-an-academic-budget","title":"How to Train BERT with an Academic Budget","date":"2021-04-15","rows_on_this_dataset":1,"code_links":4,"syntology":null},{"paper":"/paper/clear-contrastive-learning-for-sentence","title":"CLEAR: Contrastive Learning for Sentence Representation","date":"2020-12-31","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/informer-transformer-likes-informed-attention","title":"RealFormer: Transformer Likes Residual Attention","date":"2020-12-21","rows_on_this_dataset":1,"code_links":5,"syntology":null},{"paper":"/paper/mixing-adam-and-sgd-a-combined-optimization","title":"Mixing ADAM and SGD: a Combined Optimization Method","date":"2020-11-16","rows_on_this_dataset":1,"code_links":1,"syntology":null},{"paper":"/paper/a-statistical-framework-for-low-bitwidth","title":"A Statistical Framework for Low-bitwidth Training of Deep Neural Networks","date":"2020-10-27","rows_on_this_dataset":1,"code_links":2,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":4,"samples_ran":1,"samples_unverified":3,"pointer_only_for_licence":1,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/big-bird-transformers-for-longer-sequences","title":"Big Bird: Transformers for Longer Sequences","date":"2020-07-28","rows_on_this_dataset":1,"code_links":14,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":15,"samples_ran":10,"samples_unverified":5,"pointer_only_for_licence":11,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/squeezebert-what-can-computer-vision-teach","title":"SqueezeBERT: What can computer vision teach NLP about efficient neural networks?","date":"2020-06-19","rows_on_this_dataset":1,"code_links":6,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":1,"samples_ran":0,"samples_unverified":1,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/deberta-decoding-enhanced-bert-with","title":"DeBERTa: Decoding-enhanced BERT with Disentangled Attention","date":"2020-06-05","rows_on_this_dataset":1,"code_links":14,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":13,"samples_ran":4,"samples_unverified":9,"pointer_only_for_licence":3,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/synthesizer-rethinking-self-attention-in","title":"Synthesizer: Rethinking Self-Attention in Transformer Models","date":"2020-05-02","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":1,"samples_ran":1,"samples_unverified":0,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/learning-to-encode-position-for-transformer","title":"Learning to Encode Position for Transformer with Continuous Dynamical Model","date":"2020-03-13","rows_on_this_dataset":1,"code_links":1,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":6,"samples_ran":3,"samples_unverified":3,"pointer_only_for_licence":6,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/exploring-the-limits-of-transfer-learning","title":"Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer","date":"2019-10-23","rows_on_this_dataset":5,"code_links":57,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":31,"samples_ran":2,"samples_unverified":29,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/q8bert-quantized-8bit-bert","title":"Q8BERT: Quantized 8Bit BERT","date":"2019-10-14","rows_on_this_dataset":1,"code_links":5,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":11,"samples_ran":3,"samples_unverified":8,"pointer_only_for_licence":3,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/distilbert-a-distilled-version-of-bert","title":"DistilBERT, a distilled version of BERT: smaller, faster, cheaper and lighter","date":"2019-10-02","rows_on_this_dataset":1,"code_links":37,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":27,"samples_ran":19,"samples_unverified":8,"pointer_only_for_licence":0,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/albert-a-lite-bert-for-self-supervised","title":"ALBERT: A Lite BERT for Self-supervised Learning of Language Representations","date":"2019-09-26","rows_on_this_dataset":1,"code_links":48,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":126,"samples_ran":46,"samples_unverified":80,"pointer_only_for_licence":22,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/190910351","title":"TinyBERT: Distilling BERT for Natural Language Understanding","date":"2019-09-23","rows_on_this_dataset":2,"code_links":10,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":4,"samples_ran":0,"samples_unverified":4,"pointer_only_for_licence":4,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/q-bert-hessian-based-ultra-low-precision","title":"Q-BERT: Hessian Based Ultra Low Precision Quantization of BERT","date":"2019-09-12","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/structbert-incorporating-language-structures","title":"StructBERT: Incorporating Language Structures into Pre-training for Deep Language Understanding","date":"2019-08-13","rows_on_this_dataset":1,"code_links":0,"syntology":null},{"paper":"/paper/ernie-20-a-continual-pre-training-framework","title":"ERNIE 2.0: A Continual Pre-training Framework for Language Understanding","date":"2019-07-29","rows_on_this_dataset":2,"code_links":3,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":1,"samples_ran":0,"samples_unverified":1,"pointer_only_for_licence":1,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/roberta-a-robustly-optimized-bert-pretraining","title":"RoBERTa: A Robustly Optimized BERT Pretraining Approach","date":"2019-07-26","rows_on_this_dataset":1,"code_links":67,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":48,"samples_ran":22,"samples_unverified":26,"pointer_only_for_licence":23,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/spanbert-improving-pre-training-by","title":"SpanBERT: Improving Pre-training by Representing and Predicting Spans","date":"2019-07-24","rows_on_this_dataset":1,"code_links":6,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":15,"samples_ran":3,"samples_unverified":12,"pointer_only_for_licence":4,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/xlnet-generalized-autoregressive-pretraining","title":"XLNet: Generalized Autoregressive Pretraining for Language Understanding","date":"2019-06-19","rows_on_this_dataset":1,"code_links":27,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":24,"samples_ran":10,"samples_unverified":14,"pointer_only_for_licence":3,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/ernie-enhanced-language-representation-with","title":"ERNIE: Enhanced Language Representation with Informative Entities","date":"2019-05-17","rows_on_this_dataset":1,"code_links":2,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":3,"samples_ran":3,"samples_unverified":0,"pointer_only_for_licence":1,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/multi-task-deep-neural-networks-for-natural","title":"Multi-Task Deep Neural Networks for Natural Language Understanding","date":"2019-01-31","rows_on_this_dataset":1,"code_links":7,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":13,"samples_ran":5,"samples_unverified":8,"pointer_only_for_licence":2,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}},{"paper":"/paper/bert-pre-training-of-deep-bidirectional","title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding","date":"2018-10-11","rows_on_this_dataset":1,"code_links":534,"syntology":{"read_at":"2026-09-24T18:15:14+00:00","samples_harvested":659,"samples_ran":204,"samples_unverified":455,"pointer_only_for_licence":149,"claim":"Per-sample execution on synthesized fixtures; not a correctness claim."}}],"syntology_totals":{"read_at":"2026-09-24T18:15:14+00:00","papers_with_samples":24,"samples_harvested":1032,"samples_ran":351,"samples_unverified":681,"pointer_only_for_licence":238,"papers_with_no_sample_that_ran":4,"note":"the per-paper counts above, summed; not a rate"},"papers_note":"The archive never published its papers-using-dataset list; these are papers with a leaderboard row on this dataset's benchmarks."}