{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/paragraph-based-transformer-pre-training-for","title":"Paragraph-based Transformer Pre-training for Multi-Sentence Inference","arxiv_id":"2205.01228","date":"2022-05-02","proceeding":"NAACL 2022 7","authors":["Luca Di Liello","Siddhant Garg","Luca Soldaini","Alessandro Moschitti"],"abstract":"Inference tasks such as answer sentence selection (AS2) or fact verification are typically solved by fine-tuning transformer-based models as individual sentence-pair classifiers. Recent studies show that these tasks benefit from modeling dependencies across multiple candidate sentences jointly. In this paper, we first show that popular pre-trained transformers perform poorly when used for fine-tuning on multi-candidate inference tasks. We then propose a new pre-training objective that models the paragraph-level semantics across multiple input sentences. Our evaluation on three AS2 and one fact verification datasets demonstrates the superiority of our pre-training technique over the traditional ones for transformers used as joint models for multi-candidate inference tasks, as well as when used as cross-encoders for sentence-pair formulations of these tasks. Our code and pre-trained models are released at https://github.com/amazon-research/wqa-multi-sentence-inference .","url_abs":"https://arxiv.org/abs/2205.01228v2","url_pdf":"https://arxiv.org/pdf/2205.01228v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"paragraph-based-transformer-pre-training-for","repo_url":"https://github.com/amazon-research/wqa-multi-sentence-inference","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"NOASSERTION"}}],"tasks":[{"task_slug":"answer-selection","task_name":"Answer Selection"},{"task_slug":"fact-verification","task_name":"Fact Verification"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"sentence","task_name":"Sentence"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/answer-selection-on-asnq","task":"Answer Selection","dataset":"ASNQ","model":"RoBERTa-Base Joint MSPP","rank_in_archive_order":3,"of":3,"metrics":{"MAP":"0.673","MRR":"0.737"},"uses_additional_data":false},{"leaderboard":"/sota/fact-verification-on-fever","task":"Fact Verification","dataset":"FEVER","model":"RoBERTa-Base Joint MSPP Flexible","rank_in_archive_order":3,"of":7,"metrics":{"Accuracy":"75.36"},"uses_additional_data":false},{"leaderboard":"/sota/fact-verification-on-fever","task":"Fact Verification","dataset":"FEVER","model":"RoBERTa-Base Joint MSPP","rank_in_archive_order":4,"of":7,"metrics":{"Accuracy":"74.39"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-trecqa","task":"Question Answering","dataset":"TrecQA","model":"RoBERTa-Base Joint + MSPP","rank_in_archive_order":6,"of":13,"metrics":{"MAP":"0.911","MRR":"0.952"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-wikiqa","task":"Question Answering","dataset":"WikiQA","model":"RoBERTa-Base Joint MSPP","rank_in_archive_order":6,"of":25,"metrics":{"MAP":"0.887","MRR":"0.900"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2205.01228","mcp":null,"developers":"https://syntology.ai/developers"},"arxiv_metadata":null,"syntology_extracted_results":null}