{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/agent-as-judge-for-factual-summarization-of","title":"Agent-as-Judge for Factual Summarization of Long Narratives","arxiv_id":"2501.09993","date":"2025-01-17","proceeding":null,"authors":["Yeonseok Jeong","Minsoo Kim","Seung-won Hwang","Byung-Hak Kim"],"abstract":"Large Language Models (LLMs) have demonstrated near-human performance in summarization tasks based on traditional metrics such as ROUGE and BERTScore. However, these metrics do not adequately capture critical aspects of summarization quality, such as factual accuracy, particularly for long narratives (>100K tokens). Recent advances, such as LLM-as-a-Judge, address the limitations of metrics based on lexical similarity but still exhibit factual inconsistencies, especially in understanding character relationships and states. In this work, we introduce NarrativeFactScore, a novel \"Agent-as-a-Judge\" framework for evaluating and refining summaries. By leveraging a Character Knowledge Graph (CKG) extracted from input and generated summaries, NarrativeFactScore assesses the factual consistency and provides actionable guidance for refinement, such as identifying missing or erroneous facts. We demonstrate the effectiveness of NarrativeFactScore through a detailed workflow illustration and extensive validation on widely adopted benchmarks, achieving superior performance compared to competitive methods. Our results highlight the potential of agent-driven evaluation systems to improve the factual reliability of LLM-generated summaries.","url_abs":"https://arxiv.org/abs/2501.09993v1","url_pdf":"https://arxiv.org/pdf/2501.09993v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"agent-as-judge-for-factual-summarization-of","repo_url":"https://github.com/yeonseokjeong/narrativefactscore","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"long-form-narrative-summarization","task_name":"Long-Form Narrative Summarization"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/long-form-narrative-summarization-on-mensa","task":"Long-Form Narrative Summarization","dataset":"MENSA","model":"Hierarchically Merging and Agent Refinement","rank_in_archive_order":3,"of":10,"metrics":{"BERTScore (F1)":"60.22","ROUGE-1":"31.31","ROUGE-2":"8.81","ROUGE-L":"18.62"},"uses_additional_data":false},{"leaderboard":"/sota/long-form-narrative-summarization-on-moviesum","task":"Long-Form Narrative Summarization","dataset":"MovieSum","model":"Hierarchically Merging and Agent Refinement","rank_in_archive_order":2,"of":5,"metrics":{"BERTScore (F1)":"59.32","ROUGE-1":"31.31","ROUGE-2":"8.81","ROUGE-L":"18.62"},"uses_additional_data":false}],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2501.09993","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2501.09993"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/YeonseokJeong/NarrativeFactScore","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"unverified":13},"by_repo_kind":{"official":{"samples":13,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"27358d1c0a6c53b6","entry":"break_down2scenes","repo":"YeonseokJeong/NarrativeFactScore","repo_kind":"official","path":"src/fact/utils.py","file_url":"https://github.com/YeonseokJeong/NarrativeFactScore/blob/HEAD/src/fact/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"27358d1c0a6c53b6"}},{"code_sha256_prefix":"27c939504cb861df","entry":"build_fact_prompt","repo":"YeonseokJeong/NarrativeFactScore","repo_kind":"official","path":"src/fact/prompt.py","file_url":"https://github.com/YeonseokJeong/NarrativeFactScore/blob/HEAD/src/fact/prompt.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"27c939504cb861df"}},{"code_sha256_prefix":"a995b01f880808ea","entry":"calculate_bert_score","repo":"YeonseokJeong/NarrativeFactScore","repo_kind":"official","path":"calculate_metrics.py","file_url":"https://github.com/YeonseokJeong/NarrativeFactScore/blob/HEAD/calculate_metrics.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a995b01f880808ea"}},{"code_sha256_prefix":"50254ec8b6e5f2d1","entry":"calculate_bleu","repo":"YeonseokJeong/NarrativeFactScore","repo_kind":"official","path":"calculate_metrics.py","file_url":"https://github.com/YeonseokJeong/NarrativeFactScore/blob/HEAD/calculate_metrics.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"50254ec8b6e5f2d1"}},{"code_sha256_prefix":"1ac357bf2eed797a","entry":"calculate_rouge","repo":"YeonseokJeong/NarrativeFactScore","repo_kind":"official","path":"calculate_metrics.py","file_url":"https://github.com/YeonseokJeong/NarrativeFactScore/blob/HEAD/calculate_metrics.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1ac357bf2eed797a"}},{"code_sha256_prefix":"2a13a9eae8815574","entry":"delete_special","repo":"YeonseokJeong/NarrativeFactScore","repo_kind":"official","path":"src/fact/utils.py","file_url":"https://github.com/YeonseokJeong/NarrativeFactScore/blob/HEAD/src/fact/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2a13a9eae8815574"}},{"code_sha256_prefix":"d24be1d4cdb47e5f","entry":"generate_knowledge_graph_for_scripts","repo":"YeonseokJeong/NarrativeFactScore","repo_kind":"official","path":"src/kg/generate_kg.py","file_url":"https://github.com/YeonseokJeong/NarrativeFactScore/blob/HEAD/src/kg/generate_kg.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d24be1d4cdb47e5f"}},{"code_sha256_prefix":"f50f1c74536e69c4","entry":"generate_names_graph","repo":"YeonseokJeong/NarrativeFactScore","repo_kind":"official","path":"src/kg/knowledge_graph.py","file_url":"https://github.com/YeonseokJeong/NarrativeFactScore/blob/HEAD/src/kg/knowledge_graph.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f50f1c74536e69c4"}},{"code_sha256_prefix":"00ab0223fa638a2f","entry":"gpt_inference","repo":"YeonseokJeong/NarrativeFactScore","repo_kind":"official","path":"src/kg/generate_kg.py","file_url":"https://github.com/YeonseokJeong/NarrativeFactScore/blob/HEAD/src/kg/generate_kg.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"00ab0223fa638a2f"}},{"code_sha256_prefix":"c8df89ad35127fe7","entry":"load_knowledge_graph","repo":"YeonseokJeong/NarrativeFactScore","repo_kind":"official","path":"src/kg/generate_kg.py","file_url":"https://github.com/YeonseokJeong/NarrativeFactScore/blob/HEAD/src/kg/generate_kg.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c8df89ad35127fe7"}},{"code_sha256_prefix":"4308a1d6439f3b09","entry":"parse_response_text","repo":"YeonseokJeong/NarrativeFactScore","repo_kind":"official","path":"src/kg/knowledge_graph.py","file_url":"https://github.com/YeonseokJeong/NarrativeFactScore/blob/HEAD/src/kg/knowledge_graph.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4308a1d6439f3b09"}},{"code_sha256_prefix":"8936fbeddd8acbb0","entry":"remove_number_prefix","repo":"YeonseokJeong/NarrativeFactScore","repo_kind":"official","path":"src/kg/knowledge_graph.py","file_url":"https://github.com/YeonseokJeong/NarrativeFactScore/blob/HEAD/src/kg/knowledge_graph.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8936fbeddd8acbb0"}},{"code_sha256_prefix":"69d99f943fa8a011","entry":"split_and_clean_script","repo":"YeonseokJeong/NarrativeFactScore","repo_kind":"official","path":"down_data.py","file_url":"https://github.com/YeonseokJeong/NarrativeFactScore/blob/HEAD/down_data.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"69d99f943fa8a011"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}