{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/unsupervised-fact-verification-by-language","title":"Unsupervised Pretraining for Fact Verification by Language Model Distillation","arxiv_id":"2309.16540","date":"2023-09-28","proceeding":null,"authors":["Adrián Bazaga","Pietro Liò","Gos Micklem"],"abstract":"Fact verification aims to verify a claim using evidence from a trustworthy knowledge base. To address this challenge, algorithms must produce features for every claim that are both semantically meaningful, and compact enough to find a semantic alignment with the source information. In contrast to previous work, which tackled the alignment problem by learning over annotated corpora of claims and their corresponding labels, we propose SFAVEL (Self-supervised Fact Verification via Language Model Distillation), a novel unsupervised pretraining framework that leverages pre-trained language models to distil self-supervised features into high-quality claim-fact alignments without the need for annotations. This is enabled by a novel contrastive loss function that encourages features to attain high-quality claim and evidence alignments whilst preserving the semantic relationships across the corpora. Notably, we present results that achieve a new state-of-the-art on FB15k-237 (+5.3% Hits@1) and FEVER (+8% accuracy) with linear evaluation.","url_abs":"https://arxiv.org/abs/2309.16540v3","url_pdf":"https://arxiv.org/pdf/2309.16540v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"unsupervised-fact-verification-by-language","repo_url":"https://github.com/adrianbzg/sfavel","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"fact-verification","task_name":"Fact Verification"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"linear-evaluation","task_name":"Linear evaluation"}],"methods":[{"method_slug":"base","method_name":"BASE"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2309.16540","atlas_url":"https://app.syntology.ai/?focus=2309.16540","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2309.16540"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/AdrianBZG/SFAVEL","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/adrianbzg/sfavel","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":3,"unverified":1},"by_repo_kind":{"official":{"samples":4,"ran":3,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"e8819ee750c418d6","entry":"batch_collate_function","repo":"AdrianBZG/SFAVEL","repo_kind":"official","path":"src/dataset/data_handling.py","file_url":"https://github.com/AdrianBZG/SFAVEL/blob/HEAD/src/dataset/data_handling.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e8819ee750c418d6"}},{"code_sha256_prefix":"fc2d1d692a8eb198","entry":"get_huggingface_model","repo":"AdrianBZG/SFAVEL","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/AdrianBZG/SFAVEL/blob/HEAD/src/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fc2d1d692a8eb198"}},{"code_sha256_prefix":"b7def069f21916ca","entry":"get_model_size","repo":"AdrianBZG/SFAVEL","repo_kind":"official","path":"src/utils.py","file_url":"https://github.com/AdrianBZG/SFAVEL/blob/HEAD/src/utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b7def069f21916ca"}},{"code_sha256_prefix":"4100d374559366d7","entry":"run_evaluate","repo":"AdrianBZG/SFAVEL","repo_kind":"official","path":"src/finetuning.py","file_url":"https://github.com/AdrianBZG/SFAVEL/blob/HEAD/src/finetuning.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4100d374559366d7"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}