{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/summac-re-visiting-nli-based-models-for","title":"SummaC: Re-Visiting NLI-based Models for Inconsistency Detection in Summarization","arxiv_id":"2111.09525","date":"2021-11-18","proceeding":null,"authors":["Philippe Laban","Tobias Schnabel","Paul N. Bennett","Marti A. Hearst"],"abstract":"In the summarization domain, a key requirement for summaries is to be factually consistent with the input document. Previous work has found that natural language inference (NLI) models do not perform competitively when applied to inconsistency detection. In this work, we revisit the use of NLI for inconsistency detection, finding that past work suffered from a mismatch in input granularity between NLI datasets (sentence-level), and inconsistency detection (document level). We provide a highly effective and light-weight method called SummaCConv that enables NLI models to be successfully used for this task by segmenting documents into sentence units and aggregating scores between pairs of sentences. On our newly introduced benchmark called SummaC (Summary Consistency) consisting of six large inconsistency detection datasets, SummaCConv obtains state-of-the-art results with a balanced accuracy of 74.4%, a 5% point improvement compared to prior work. We make the models and datasets available: https://github.com/tingofurro/summac","url_abs":"https://arxiv.org/abs/2111.09525v1","url_pdf":"https://arxiv.org/pdf/2111.09525v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"summac-re-visiting-nli-based-models-for","repo_url":"https://github.com/tingofurro/summac","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":null},{"paper_slug":"summac-re-visiting-nli-based-models-for","repo_url":"https://github.com/salesforce/auditnlg","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"BSD-3-Clause"}},{"paper_slug":"summac-re-visiting-nli-based-models-for","repo_url":"https://github.com/stair-lab/villm-eval","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"natural-language-inference","task_name":"Natural Language Inference"},{"task_slug":"sentence","task_name":"Sentence"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2111.09525","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2111.09525"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/tingofurro/summac","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/salesforce/auditnlg","reach":{"status":"ok","spdx":"BSD-3-Clause"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/stair-lab/villm-eval","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":2,"ran_honours":1,"unverified":20},"by_repo_kind":{"official":{"samples":3,"ran":3,"repositories":1},"listed":{"samples":20,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"db7f9bf2c56eba82","entry":"card_to_name","repo":"tingofurro/summac","repo_kind":"official","path":"summac/model_summac.py","file_url":"https://github.com/tingofurro/summac/blob/HEAD/summac/model_summac.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"db7f9bf2c56eba82"}},{"code_sha256_prefix":"ea7a5375f9985efb","entry":"get_neutral_idx","repo":"tingofurro/summac","repo_kind":"official","path":"summac/model_summac.py","file_url":"https://github.com/tingofurro/summac/blob/HEAD/summac/model_summac.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"ea7a5375f9985efb"}},{"code_sha256_prefix":"336a19c3f0f872cf","entry":"name_to_card","repo":"tingofurro/summac","repo_kind":"official","path":"summac/model_summac.py","file_url":"https://github.com/tingofurro/summac/blob/HEAD/summac/model_summac.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"336a19c3f0f872cf"}},{"code_sha256_prefix":"75b06289edde523d","entry":"chatgpt_score","repo":"salesforce/auditnlg","repo_kind":"listed","path":"src/auditnlg/factualness/classifier.py","file_url":"https://github.com/salesforce/auditnlg/blob/HEAD/src/auditnlg/factualness/classifier.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"75b06289edde523d"}},{"code_sha256_prefix":"4f684528e060f684","entry":"constraint_scores","repo":"salesforce/auditnlg","repo_kind":"listed","path":"src/auditnlg/constraint/exam.py","file_url":"https://github.com/salesforce/auditnlg/blob/HEAD/src/auditnlg/constraint/exam.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"4f684528e060f684"}},{"code_sha256_prefix":"4920d301e2e44bc8","entry":"create_model","repo":"salesforce/auditnlg","repo_kind":"listed","path":"src/auditnlg/regeneration/modeling.py","file_url":"https://github.com/salesforce/auditnlg/blob/HEAD/src/auditnlg/regeneration/modeling.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"4920d301e2e44bc8"}},{"code_sha256_prefix":"9c90cb1632873979","entry":"evaluate_2step_prompt","repo":"salesforce/auditnlg","repo_kind":"listed","path":"src/auditnlg/constraint/constraint_checker.py","file_url":"https://github.com/salesforce/auditnlg/blob/HEAD/src/auditnlg/constraint/constraint_checker.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"9c90cb1632873979"}},{"code_sha256_prefix":"b6e987aeac696a8d","entry":"example_format_checker","repo":"salesforce/auditnlg","repo_kind":"listed","path":"src/auditnlg/utils.py","file_url":"https://github.com/salesforce/auditnlg/blob/HEAD/src/auditnlg/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"b6e987aeac696a8d"}},{"code_sha256_prefix":"cb8c3be6c3abff92","entry":"factual_scores","repo":"salesforce/auditnlg","repo_kind":"listed","path":"src/auditnlg/factualness/exam.py","file_url":"https://github.com/salesforce/auditnlg/blob/HEAD/src/auditnlg/factualness/exam.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"cb8c3be6c3abff92"}},{"code_sha256_prefix":"db47fe0250244595","entry":"format_model_inputs","repo":"salesforce/auditnlg","repo_kind":"listed","path":"src/auditnlg/constraint/constraint_checker.py","file_url":"https://github.com/salesforce/auditnlg/blob/HEAD/src/auditnlg/constraint/constraint_checker.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"db47fe0250244595"}},{"code_sha256_prefix":"b2fd3d901d235116","entry":"format_model_inputs","repo":"salesforce/auditnlg","repo_kind":"listed","path":"src/auditnlg/regeneration/prompt_helper.py","file_url":"https://github.com/salesforce/auditnlg/blob/HEAD/src/auditnlg/regeneration/prompt_helper.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"b2fd3d901d235116"}},{"code_sha256_prefix":"7a7e60f86ff3c48c","entry":"get_global_knowledge","repo":"salesforce/auditnlg","repo_kind":"listed","path":"src/auditnlg/utils.py","file_url":"https://github.com/salesforce/auditnlg/blob/HEAD/src/auditnlg/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"7a7e60f86ff3c48c"}},{"code_sha256_prefix":"e8bd306b2dea16ca","entry":"get_instruction","repo":"salesforce/auditnlg","repo_kind":"listed","path":"src/auditnlg/factualness/utils.py","file_url":"https://github.com/salesforce/auditnlg/blob/HEAD/src/auditnlg/factualness/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"e8bd306b2dea16ca"}},{"code_sha256_prefix":"16b67432f36fbab3","entry":"gpt3_score","repo":"salesforce/auditnlg","repo_kind":"listed","path":"src/auditnlg/factualness/classifier.py","file_url":"https://github.com/salesforce/auditnlg/blob/HEAD/src/auditnlg/factualness/classifier.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"16b67432f36fbab3"}},{"code_sha256_prefix":"bffc6c96cc1984d2","entry":"is_better_candidate","repo":"salesforce/auditnlg","repo_kind":"listed","path":"src/auditnlg/regeneration/prompt_helper.py","file_url":"https://github.com/salesforce/auditnlg/blob/HEAD/src/auditnlg/regeneration/prompt_helper.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"bffc6c96cc1984d2"}},{"code_sha256_prefix":"a3a1bbfb4a4b4a0a","entry":"knowledge_parser","repo":"salesforce/auditnlg","repo_kind":"listed","path":"src/auditnlg/utils.py","file_url":"https://github.com/salesforce/auditnlg/blob/HEAD/src/auditnlg/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"a3a1bbfb4a4b4a0a"}},{"code_sha256_prefix":"86b13ed40996b6eb","entry":"localllm_score","repo":"salesforce/auditnlg","repo_kind":"listed","path":"src/auditnlg/factualness/classifier.py","file_url":"https://github.com/salesforce/auditnlg/blob/HEAD/src/auditnlg/factualness/classifier.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"86b13ed40996b6eb"}},{"code_sha256_prefix":"3f5b2ebc7935f2fe","entry":"openai_moderation_API","repo":"salesforce/auditnlg","repo_kind":"listed","path":"src/auditnlg/safety/classifier.py","file_url":"https://github.com/salesforce/auditnlg/blob/HEAD/src/auditnlg/safety/classifier.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"3f5b2ebc7935f2fe"}},{"code_sha256_prefix":"cd25f07d2d7a51c1","entry":"openai_score","repo":"salesforce/auditnlg","repo_kind":"listed","path":"src/auditnlg/safety/classifier.py","file_url":"https://github.com/salesforce/auditnlg/blob/HEAD/src/auditnlg/safety/classifier.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"cd25f07d2d7a51c1"}},{"code_sha256_prefix":"b4bd6f6d1ef96d82","entry":"perspective_API","repo":"salesforce/auditnlg","repo_kind":"listed","path":"src/auditnlg/safety/classifier.py","file_url":"https://github.com/salesforce/auditnlg/blob/HEAD/src/auditnlg/safety/classifier.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"b4bd6f6d1ef96d82"}},{"code_sha256_prefix":"337b87c313053f73","entry":"preprocess_knowledge","repo":"salesforce/auditnlg","repo_kind":"listed","path":"src/auditnlg/factualness/utils.py","file_url":"https://github.com/salesforce/auditnlg/blob/HEAD/src/auditnlg/factualness/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"337b87c313053f73"}},{"code_sha256_prefix":"1fe3eb4502d7bb94","entry":"rerun_evaluations","repo":"salesforce/auditnlg","repo_kind":"listed","path":"src/auditnlg/regeneration/prompt_helper.py","file_url":"https://github.com/salesforce/auditnlg/blob/HEAD/src/auditnlg/regeneration/prompt_helper.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"1fe3eb4502d7bb94"}},{"code_sha256_prefix":"ff20927fe6849434","entry":"retrieve_global_knowledge","repo":"salesforce/auditnlg","repo_kind":"listed","path":"src/auditnlg/factualness/utils.py","file_url":"https://github.com/salesforce/auditnlg/blob/HEAD/src/auditnlg/factualness/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"ff20927fe6849434"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}