{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/zero-shot-fact-verification-via-natural-logic","title":"Zero-Shot Fact Verification via Natural Logic and Large Language Models","arxiv_id":"2410.03341","date":"2024-10-04","proceeding":null,"authors":["Marek Strong","Rami Aly","Andreas Vlachos"],"abstract":"The recent development of fact verification systems with natural logic has enhanced their explainability by aligning claims with evidence through set-theoretic operators, providing faithful justifications. Despite these advancements, such systems often rely on a large amount of training data annotated with natural logic. To address this issue, we propose a zero-shot method that utilizes the generalization capabilities of instruction-tuned large language models. To comprehensively assess the zero-shot capabilities of our method and other fact verification systems, we evaluate all models on both artificial and real-world claims, including multilingual datasets. We also compare our method against other fact verification systems in two setups. First, in the zero-shot generalization setup, we demonstrate that our approach outperforms other systems that were not specifically trained on natural logic data, achieving an average accuracy improvement of 8.96 points over the best-performing baseline. Second, in the zero-shot transfer setup, we show that current systems trained on natural logic data do not generalize well to other domains, and our method outperforms these systems across all datasets with real-world claims.","url_abs":"https://arxiv.org/abs/2410.03341v1","url_pdf":"https://arxiv.org/pdf/2410.03341v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"zero-shot-fact-verification-via-natural-logic","repo_url":"https://github.com/marekstrong/Zero-NatVer","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"AGPL-3.0"}}],"tasks":[{"task_slug":"fact-verification","task_name":"Fact Verification"},{"task_slug":"zero-shot-generalization","task_name":"Zero-shot Generalization"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2410.03341","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2410.03341"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/marekstrong/Zero-NatVer","reach":{"status":"ok","spdx":"AGPL-3.0"}}],"summary":{"ran":10},"by_repo_kind":{"official":{"samples":10,"ran":10,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":10,"samples":[{"code_sha256_prefix":"f7647e182153b9f7","entry":"build_template_align","repo":"marekstrong/Zero-NatVer","repo_kind":"official","path":"prompts/prompt_templates.py","file_url":"https://github.com/marekstrong/Zero-NatVer/blob/HEAD/prompts/prompt_templates.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"f7647e182153b9f7"}},{"code_sha256_prefix":"b5ec5466294314d3","entry":"build_template_chunk","repo":"marekstrong/Zero-NatVer","repo_kind":"official","path":"prompts/prompt_templates.py","file_url":"https://github.com/marekstrong/Zero-NatVer/blob/HEAD/prompts/prompt_templates.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"b5ec5466294314d3"}},{"code_sha256_prefix":"2f504540f3b178e0","entry":"extract_first_quoted_text","repo":"marekstrong/Zero-NatVer","repo_kind":"official","path":"alignment.py","file_url":"https://github.com/marekstrong/Zero-NatVer/blob/HEAD/alignment.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"2f504540f3b178e0"}},{"code_sha256_prefix":"8d892a74ab9fcd4a","entry":"get_logger","repo":"marekstrong/Zero-NatVer","repo_kind":"official","path":"log_config.py","file_url":"https://github.com/marekstrong/Zero-NatVer/blob/HEAD/log_config.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"8d892a74ab9fcd4a"}},{"code_sha256_prefix":"36c99317c123d80d","entry":"get_only_alnumspace","repo":"marekstrong/Zero-NatVer","repo_kind":"official","path":"utils.py","file_url":"https://github.com/marekstrong/Zero-NatVer/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"36c99317c123d80d"}},{"code_sha256_prefix":"955342af83f4fd47","entry":"get_templates_ENTAILMENT","repo":"marekstrong/Zero-NatVer","repo_kind":"official","path":"prompts/prompt_natops.py","file_url":"https://github.com/marekstrong/Zero-NatVer/blob/HEAD/prompts/prompt_natops.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"955342af83f4fd47"}},{"code_sha256_prefix":"449a842046ac91db","entry":"get_templates_EQUIVALENCE","repo":"marekstrong/Zero-NatVer","repo_kind":"official","path":"prompts/prompt_natops.py","file_url":"https://github.com/marekstrong/Zero-NatVer/blob/HEAD/prompts/prompt_natops.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"449a842046ac91db"}},{"code_sha256_prefix":"a82256c7a05fe38c","entry":"get_templates_NEGATION","repo":"marekstrong/Zero-NatVer","repo_kind":"official","path":"prompts/prompt_natops.py","file_url":"https://github.com/marekstrong/Zero-NatVer/blob/HEAD/prompts/prompt_natops.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"a82256c7a05fe38c"}},{"code_sha256_prefix":"877e4b9e3be5068c","entry":"load_json","repo":"marekstrong/Zero-NatVer","repo_kind":"official","path":"utils.py","file_url":"https://github.com/marekstrong/Zero-NatVer/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"877e4b9e3be5068c"}},{"code_sha256_prefix":"adad1a6268ed202e","entry":"load_jsonl","repo":"marekstrong/Zero-NatVer","repo_kind":"official","path":"utils.py","file_url":"https://github.com/marekstrong/Zero-NatVer/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"AGPL-3.0","inline_ok":false,"mcp_get_code":{"code_sha256":"adad1a6268ed202e"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}