{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/a-training-free-and-reference-free","title":"A Training-free and Reference-free Summarization Evaluation Metric via Centrality-weighted Relevance and Self-referenced Redundancy","arxiv_id":"2106.13945","date":"2021-06-26","proceeding":"ACL 2021 5","authors":["Wang Chen","Piji Li","Irwin King"],"abstract":"In recent years, reference-based and supervised summarization evaluation metrics have been widely explored. However, collecting human-annotated references and ratings are costly and time-consuming. To avoid these limitations, we propose a training-free and reference-free summarization evaluation metric. Our metric consists of a centrality-weighted relevance score and a self-referenced redundancy score. The relevance score is computed between the pseudo reference built from the source document and the given summary, where the pseudo reference content is weighted by the sentence centrality to provide importance guidance. Besides an $F_1$-based relevance score, we also design an $F_\\beta$-based variant that pays more attention to the recall score. As for the redundancy score of the summary, we compute a self-masked similarity score with the summary itself to evaluate the redundant information in the summary. Finally, we combine the relevance and redundancy scores to produce the final evaluation score of the given summary. Extensive experiments show that our methods can significantly outperform existing methods on both multi-document and single-document summarization evaluation.","url_abs":"https://arxiv.org/abs/2106.13945v1","url_pdf":"https://arxiv.org/pdf/2106.13945v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"a-training-free-and-reference-free","repo_url":"https://github.com/Chen-Wang-CUHK/Training-Free-and-Ref-Free-Summ-Evaluation","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"document-summarization","task_name":"Document Summarization"},{"task_slug":"sentence","task_name":"Sentence"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2106.13945","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2106.13945"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/Chen-Wang-CUHK/Training-Free-and-Ref-Free-Summ-Evaluation","reach":null}],"summary":{"ran_draft_wrong":1,"ran_honours":1},"by_repo_kind":{"official":{"samples":2,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":2,"samples":[{"code_sha256_prefix":"d9031c5dd2519311","entry":"get_idf","repo":"Chen-Wang-CUHK/Training-Free-and-Ref-Free-Summ-Evaluation","repo_kind":"official","path":"ref_free_metrics/similarity_measurements/supert_global_idf_renormalize.py","file_url":"https://github.com/Chen-Wang-CUHK/Training-Free-and-Ref-Free-Summ-Evaluation/blob/HEAD/ref_free_metrics/similarity_measurements/supert_global_idf_renormalize.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d9031c5dd2519311"}},{"code_sha256_prefix":"5d4d1bd77c54256c","entry":"get_my_score","repo":"Chen-Wang-CUHK/Training-Free-and-Ref-Free-Summ-Evaluation","repo_kind":"official","path":"ref_free_metrics/similarity_measurements/supert_global_idf_renormalize.py","file_url":"https://github.com/Chen-Wang-CUHK/Training-Free-and-Ref-Free-Summ-Evaluation/blob/HEAD/ref_free_metrics/similarity_measurements/supert_global_idf_renormalize.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5d4d1bd77c54256c"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}