{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/text-to-text-pre-training-for-data-to-text","title":"Text-to-Text Pre-Training for Data-to-Text Tasks","arxiv_id":"2005.10433","date":"2020-05-21","proceeding":"INLG (ACL) 2020 12","authors":["Mihir Kale","Abhinav Rastogi"],"abstract":"We study the pre-train + fine-tune strategy for data-to-text tasks. Our experiments indicate that text-to-text pre-training in the form of T5, enables simple, end-to-end transformer based models to outperform pipelined neural architectures tailored for data-to-text generation, as well as alternative language model based pre-training techniques such as BERT and GPT-2. Importantly, T5 pre-training leads to better generalization, as evidenced by large improvements on out-of-domain test sets. We hope our work serves as a useful baseline for future research, as transfer learning becomes ever more prevalent for data-to-text tasks.","url_abs":"https://arxiv.org/abs/2005.10433v3","url_pdf":"https://arxiv.org/pdf/2005.10433v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"text-to-text-pre-training-for-data-to-text","repo_url":"https://github.com/google-research-datasets/ToTTo","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"text-to-text-pre-training-for-data-to-text","repo_url":"https://github.com/shark-nlp/cont","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"data-to-text-generation","task_name":"Data-to-Text Generation"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"text-generation","task_name":"Text Generation"},{"task_slug":"transfer-learning","task_name":"Transfer Learning"}],"methods":[{"method_slug":"adafactor","method_name":"Adafactor"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"glu","method_name":"Gated Linear Unit"},{"method_slug":"inverse-square-root-schedule","method_name":"Inverse Square Root Schedule"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"sentencepiece","method_name":"SentencePiece"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"t5","method_name":"T5"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/data-to-text-generation-on-multiwoz-2-1","task":"Data-to-Text Generation","dataset":"MULTIWOZ 2.1","model":"T5-Base","rank_in_archive_order":1,"of":5,"metrics":{"BLEU":"35.1"},"uses_additional_data":false},{"leaderboard":"/sota/data-to-text-generation-on-totto","task":"Data-to-Text Generation","dataset":"ToTTo","model":"T5-3B","rank_in_archive_order":1,"of":6,"metrics":{"BLEU":"49.5","PARENT":"58.4"},"uses_additional_data":false},{"leaderboard":"/sota/data-to-text-generation-on-webnlg","task":"Data-to-Text Generation","dataset":"WebNLG","model":"T5-Base","rank_in_archive_order":11,"of":20,"metrics":{"BLEU":"64.7"},"uses_additional_data":false},{"leaderboard":"/sota/data-to-text-generation-on-webnlg-full-1","task":"Data-to-Text Generation","dataset":"WebNLG Full","model":"T5-Large","rank_in_archive_order":5,"of":8,"metrics":{"BLEU":"57.1"},"uses_additional_data":false}],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2005.10433","mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}