{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/themis-towards-flexible-and-interpretable-nlg","title":"Themis: A Reference-free NLG Evaluation Language Model with Flexibility and Interpretability","arxiv_id":"2406.18365","date":"2024-06-26","proceeding":null,"authors":["Xinyu Hu","Li Lin","Mingqi Gao","Xunjian Yin","Xiaojun Wan"],"abstract":"The evaluation of natural language generation (NLG) tasks is a significant and longstanding research area. With the recent emergence of powerful large language models (LLMs), some studies have turned to LLM-based automatic evaluation methods, which demonstrate great potential to become a new evaluation paradigm following traditional string-based and model-based metrics. However, despite the improved performance of existing methods, they still possess some deficiencies, such as dependency on references and limited evaluation flexibility. Therefore, in this paper, we meticulously construct a large-scale NLG evaluation corpus NLG-Eval with annotations from both human and GPT-4 to alleviate the lack of relevant data in this field. Furthermore, we propose Themis, an LLM dedicated to NLG evaluation, which has been trained with our designed multi-perspective consistency verification and rating-oriented preference alignment methods. Themis can conduct flexible and interpretable evaluations without references, and it exhibits superior evaluation performance on various NLG tasks, simultaneously generalizing well to unseen tasks and surpassing other evaluation models, including GPT-4.","url_abs":"https://arxiv.org/abs/2406.18365v2","url_pdf":"https://arxiv.org/pdf/2406.18365v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"themis-towards-flexible-and-interpretable-nlg","repo_url":"https://github.com/PKU-ONELab/Themis","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":null}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"text-generation","task_name":"Text Generation"},{"task_slug":"nlg-evaluation","task_name":"nlg evaluation"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt-4","method_name":"GPT-4"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2406.18365","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.18365"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/PKU-ONELab/Themis","reach":null}],"summary":{"ran":3,"ran_fixture":2},"by_repo_kind":{"official":{"samples":5,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"552e96a0b9c81fe5","entry":"TauOptimizationResult","repo":"PKU-ONELab/Themis","repo_kind":"official","path":"tau_optimization.py","file_url":"https://github.com/PKU-ONELab/Themis/blob/HEAD/tau_optimization.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"552e96a0b9c81fe5"}},{"code_sha256_prefix":"20c1fb317434f173","entry":"TauSufficientStats","repo":"PKU-ONELab/Themis","repo_kind":"official","path":"tau_optimization.py","file_url":"https://github.com/PKU-ONELab/Themis/blob/HEAD/tau_optimization.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"20c1fb317434f173"}},{"code_sha256_prefix":"35d010b6538ec19d","entry":"_RankedPair","repo":"PKU-ONELab/Themis","repo_kind":"official","path":"tau_optimization.py","file_url":"https://github.com/PKU-ONELab/Themis/blob/HEAD/tau_optimization.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"35d010b6538ec19d"}},{"code_sha256_prefix":"01fea23a31f8f70e","entry":"_enumerate_pairs","repo":"PKU-ONELab/Themis","repo_kind":"official","path":"tau_optimization.py","file_url":"https://github.com/PKU-ONELab/Themis/blob/HEAD/tau_optimization.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"01fea23a31f8f70e"}},{"code_sha256_prefix":"48643073cdf99320","entry":"tau_optimization","repo":"PKU-ONELab/Themis","repo_kind":"official","path":"tau_optimization.py","file_url":"https://github.com/PKU-ONELab/Themis/blob/HEAD/tau_optimization.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"48643073cdf99320"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}