{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/bleurt-learning-robust-metrics-for-text","title":"BLEURT: Learning Robust Metrics for Text Generation","arxiv_id":"2004.04696","date":"2020-04-09","proceeding":"ACL 2020 6","authors":["Thibault Sellam","Dipanjan Das","Ankur P. Parikh"],"abstract":"Text generation has made significant advances in the last few years. Yet, evaluation metrics have lagged behind, as the most popular choices (e.g., BLEU and ROUGE) may correlate poorly with human judgments. We propose BLEURT, a learned evaluation metric based on BERT that can model human judgments with a few thousand possibly biased training examples. A key aspect of our approach is a novel pre-training scheme that uses millions of synthetic examples to help the model generalize. BLEURT provides state-of-the-art results on the last three years of the WMT Metrics shared task and the WebNLG Competition dataset. In contrast to a vanilla BERT-based approach, it yields superior results even when the training data is scarce and out-of-distribution.","url_abs":"https://arxiv.org/abs/2004.04696v5","url_pdf":"https://arxiv.org/pdf/2004.04696v5.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"bleurt-learning-robust-metrics-for-text","repo_url":"https://github.com/google-research/bleurt","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"tf","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"bleurt-learning-robust-metrics-for-text","repo_url":"https://github.com/ShiYaya/Awesome_Evaluation_Metrics_for_Text_Generation","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok"}},{"paper_slug":"bleurt-learning-robust-metrics-for-text","repo_url":"https://github.com/sharanya-dasgupta001/hallushift","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"bleurt-learning-robust-metrics-for-text","repo_url":"https://github.com/thu-coai/OpenMEVA","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"tf","reach":null}],"tasks":[{"task_slug":"text-generation","task_name":"Text Generation"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2004.04696","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2004.04696"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/sharanya-dasgupta001/hallushift","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ShiYaya/Awesome_Evaluation_Metrics_for_Text_Generation","reach":{"status":"ok"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/Tiiiger/bert_score","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/thu-coai/OpenMEVA","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/google-research/bleurt","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_draft_wrong":1,"unverified":1},"by_repo_kind":{"listed":{"samples":2,"ran":1,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":1,"samples":[{"code_sha256_prefix":"bbd04de59ff2e0e2","entry":"bleurt_processing","repo":"sharanya-dasgupta001/hallushift","repo_kind":"listed","path":"functions.py","file_url":"https://github.com/sharanya-dasgupta001/hallushift/blob/HEAD/functions.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"bbd04de59ff2e0e2"}},{"code_sha256_prefix":"865f84eac22265a4","entry":"BLEURT","repo":"thu-coai/OpenMEVA","repo_kind":"listed","path":"eva/bleurt.py","file_url":"https://github.com/thu-coai/OpenMEVA/blob/HEAD/eva/bleurt.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"865f84eac22265a4"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}