{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/ernie-gen-an-enhanced-multi-flow-pre-training","title":"ERNIE-GEN: An Enhanced Multi-Flow Pre-training and Fine-tuning Framework for Natural Language Generation","arxiv_id":"2001.11314","date":"2020-01-26","proceeding":null,"authors":["Dongling Xiao","Han Zhang","Yukun Li","Yu Sun","Hao Tian","Hua Wu","Haifeng Wang"],"abstract":"Current pre-training works in natural language generation pay little attention to the problem of exposure bias on downstream tasks. To address this issue, we propose an enhanced multi-flow sequence to sequence pre-training and fine-tuning framework named ERNIE-GEN, which bridges the discrepancy between training and inference with an infilling generation mechanism and a noise-aware generation method. To make generation closer to human writing patterns, this framework introduces a span-by-span generation flow that trains the model to predict semantically-complete spans consecutively rather than predicting word by word. Unlike existing pre-training methods, ERNIE-GEN incorporates multi-granularity target sampling to construct pre-training data, which enhances the correlation between encoder and decoder. Experimental results demonstrate that ERNIE-GEN achieves state-of-the-art results with a much smaller amount of pre-training data and parameters on a range of language generation tasks, including abstractive summarization (Gigaword and CNN/DailyMail), question generation (SQuAD), dialogue generation (Persona-Chat) and generative question answering (CoQA).","url_abs":"https://arxiv.org/abs/2001.11314v3","url_pdf":"https://arxiv.org/pdf/2001.11314v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"ernie-gen-an-enhanced-multi-flow-pre-training","repo_url":"https://github.com/Sharpiless/Versailles-text-generation-with-paddlepaddle","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"paddle","reach":{"status":"unanswered"}},{"paper_slug":"ernie-gen-an-enhanced-multi-flow-pre-training","repo_url":"https://github.com/https-github-com-GiangHoang9912/ernie-gen","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"paddle","reach":{"status":"unanswered"}},{"paper_slug":"ernie-gen-an-enhanced-multi-flow-pre-training","repo_url":"https://github.com/MindCode-4/code-10/tree/main/FDDE/train-fine","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null},{"paper_slug":"ernie-gen-an-enhanced-multi-flow-pre-training","repo_url":"https://github.com/MindSpore-scientific/code-12/tree/main/FDDE/train-fine","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"mindspore","reach":null},{"paper_slug":"ernie-gen-an-enhanced-multi-flow-pre-training","repo_url":"https://github.com/PaddlePaddle/PaddleNLP/blob/develop/paddlenlp/transformers/ernie_gen/modeling.py","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"paddle","reach":null}],"tasks":[{"task_slug":"abstractive-text-summarization","task_name":"Abstractive Text Summarization"},{"task_slug":"decoder","task_name":"Decoder"},{"task_slug":"dialogue-generation","task_name":"Dialogue Generation"},{"task_slug":"generative-question-answering","task_name":"Generative Question Answering"},{"task_slug":"question-generation","task_name":"Question Generation"},{"task_slug":"question-generation","task_name":"Question-Generation"},{"task_slug":"text-generation","task_name":"Text Generation"},{"task_slug":"text-summarization","task_name":"Text Summarization"}],"methods":[{"method_slug":"ernie-gen","method_name":"ERNIE-GEN"}],"datasets_introduced":[],"methods_introduced":[{"slug":"ernie-gen","name":"ERNIE-GEN","full_name":"ERNIE-GEN"}],"results":[{"leaderboard":"/sota/abstractive-text-summarization-on-cnn-daily","task":"Abstractive Text Summarization","dataset":"CNN / Daily Mail","model":"ERNIE-GENLARGE (large-scale text corpora)","rank_in_archive_order":15,"of":53,"metrics":{"ROUGE-1":"44.31","ROUGE-2":"21.35","ROUGE-L":"41.60"},"uses_additional_data":true},{"leaderboard":"/sota/abstractive-text-summarization-on-cnn-daily","task":"Abstractive Text Summarization","dataset":"CNN / Daily Mail","model":"ERNIE-GENLARGE","rank_in_archive_order":20,"of":53,"metrics":{"ROUGE-1":"44.02","ROUGE-2":"21.17","ROUGE-L":"41.26"},"uses_additional_data":false},{"leaderboard":"/sota/abstractive-text-summarization-on-cnn-daily","task":"Abstractive Text Summarization","dataset":"CNN / Daily Mail","model":"ERNIE-GENBASE","rank_in_archive_order":26,"of":53,"metrics":{"ROUGE-1":"42.30","ROUGE-2":"19.92","ROUGE-L":"39.68"},"uses_additional_data":true},{"leaderboard":"/sota/generative-question-answering-on-coqa","task":"Generative Question Answering","dataset":"CoQA","model":"ERNIE-GEN","rank_in_archive_order":1,"of":3,"metrics":{"F1-Score":"84.5"},"uses_additional_data":true},{"leaderboard":"/sota/question-generation-on-squad11","task":"Question Generation","dataset":"SQuAD1.1","model":"ERNIE-GENLARGE (beam size=5)","rank_in_archive_order":1,"of":13,"metrics":{"BLEU-4":"25.41"},"uses_additional_data":true},{"leaderboard":"/sota/text-summarization-on-gigaword","task":"Text Summarization","dataset":"GigaWord","model":"ERNIE-GENLARGE (large-scale text corpora)","rank_in_archive_order":10,"of":41,"metrics":{"ROUGE-1":"39.46","ROUGE-2":"20.34","ROUGE-L":"36.74"},"uses_additional_data":true},{"leaderboard":"/sota/text-summarization-on-gigaword","task":"Text Summarization","dataset":"GigaWord","model":"ERNIE-GENLARGE","rank_in_archive_order":13,"of":41,"metrics":{"ROUGE-1":"39.25","ROUGE-2":"20.25","ROUGE-L":"36.53"},"uses_additional_data":true},{"leaderboard":"/sota/text-summarization-on-gigaword","task":"Text Summarization","dataset":"GigaWord","model":"ERNIE-GENBASE","rank_in_archive_order":19,"of":41,"metrics":{"ROUGE-1":"38.83","ROUGE-2":"20.04","ROUGE-L":"36.20"},"uses_additional_data":true},{"leaderboard":"/sota/text-summarization-on-gigaword-10k","task":"Text Summarization","dataset":"GigaWord-10k","model":"ERNIE-GENLARGE (large-scale text corpora)","rank_in_archive_order":1,"of":3,"metrics":{"ROUGE-1":"35.51","ROUGE-2":"16.79","ROUGE-L":"33.23"},"uses_additional_data":true},{"leaderboard":"/sota/text-summarization-on-gigaword-10k","task":"Text Summarization","dataset":"GigaWord-10k","model":"ERNIE-GENLARGE","rank_in_archive_order":2,"of":3,"metrics":{"ROUGE-1":"35.05","ROUGE-2":"16.10","ROUGE-L":"32.50"},"uses_additional_data":true},{"leaderboard":"/sota/text-summarization-on-gigaword-10k","task":"Text Summarization","dataset":"GigaWord-10k","model":"ERNIE-GENBASE","rank_in_archive_order":3,"of":3,"metrics":{"ROUGE-1":"33.75","ROUGE-2":"15.23","ROUGE-L":"31.35"},"uses_additional_data":true}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2001.11314","mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}