{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2510-13494","title":"LiteraryQA: Towards Effective Evaluation of Long-document Narrative QA","arxiv_id":"2510.13494","date":"2025-10-15","proceeding":"EMNLP","authors":["Tommaso Bonomo","Luca Gioffré","Roberto Navigli"],"abstract":"Question Answering (QA) on narrative text poses a unique challenge to current systems, requiring a deep understanding of long, complex documents. However, the reliability of NarrativeQA, the most widely used benchmark in this domain, is hindered by noisy documents and flawed QA pairs. In this work, we introduce LiteraryQA, a high-quality subset of NarrativeQA focused on literary works. Using a human- and LLM-validated pipeline, we identify and correct low-quality QA samples while removing extraneous text from source documents. We then carry out a meta-evaluation of automatic metrics to clarify how systems should be evaluated on LiteraryQA. This analysis reveals that all n-gram-based metrics have a low system-level correlation to human judgment, while LLM-as-a-Judge evaluations, even with small open-weight models, can strongly agree with the ranking identified by humans. Finally, we benchmark a set of long-context LLMs on LiteraryQA. We release our code and data at https://github.com/SapienzaNLP/LiteraryQA.","url_abs":"https://arxiv.org/abs/2510.13494","url_pdf":"https://arxiv.org/pdf/2510.13494","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2510.13494","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2510.13494"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/SapienzaNLP/LiteraryQA","reach":{"status":"ok"}}],"summary":{"unverified":6},"by_repo_kind":{"found_in_text":{"samples":6,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":6,"samples":[{"code_sha256_prefix":"b2e99f6b9eea4ae3","entry":"exact_match_score","repo":"SapienzaNLP/LiteraryQA","repo_kind":"found_in_text","path":"literaryqa/ngram_metrics.py","file_url":"https://github.com/SapienzaNLP/LiteraryQA/blob/HEAD/literaryqa/ngram_metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"b2e99f6b9eea4ae3"}},{"code_sha256_prefix":"757c1276cb093efc","entry":"f1_score","repo":"SapienzaNLP/LiteraryQA","repo_kind":"found_in_text","path":"literaryqa/ngram_metrics.py","file_url":"https://github.com/SapienzaNLP/LiteraryQA/blob/HEAD/literaryqa/ngram_metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"757c1276cb093efc"}},{"code_sha256_prefix":"eb18c22b702e08bd","entry":"get_rubric","repo":"SapienzaNLP/LiteraryQA","repo_kind":"found_in_text","path":"literaryqa/prometheus.py","file_url":"https://github.com/SapienzaNLP/LiteraryQA/blob/HEAD/literaryqa/prometheus.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"eb18c22b702e08bd"}},{"code_sha256_prefix":"95ebd8f847c607da","entry":"is_text_corrupted","repo":"SapienzaNLP/LiteraryQA","repo_kind":"found_in_text","path":"literaryqa/download.py","file_url":"https://github.com/SapienzaNLP/LiteraryQA/blob/HEAD/literaryqa/download.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"95ebd8f847c607da"}},{"code_sha256_prefix":"dfcf1201010b7d31","entry":"normalize_answer","repo":"SapienzaNLP/LiteraryQA","repo_kind":"found_in_text","path":"literaryqa/ngram_metrics.py","file_url":"https://github.com/SapienzaNLP/LiteraryQA/blob/HEAD/literaryqa/ngram_metrics.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"dfcf1201010b7d31"}},{"code_sha256_prefix":"946c1c26318b21cc","entry":"remove_gutenberg_info","repo":"SapienzaNLP/LiteraryQA","repo_kind":"found_in_text","path":"literaryqa/clean.py","file_url":"https://github.com/SapienzaNLP/LiteraryQA/blob/HEAD/literaryqa/clean.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"946c1c26318b21cc"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.CL","source":"arxiv_api"},"syntology_extracted_results":null}