{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/litsearch-a-retrieval-benchmark-for","title":"LitSearch: A Retrieval Benchmark for Scientific Literature Search","arxiv_id":"2407.18940","date":"2024-07-10","proceeding":null,"authors":["Anirudh Ajith","Mengzhou Xia","Alexis Chevalier","Tanya Goyal","Danqi Chen","Tianyu Gao"],"abstract":"Literature search questions, such as \"Where can I find research on the evaluation of consistency in generated summaries?\" pose significant challenges for modern search engines and retrieval systems. These questions often require a deep understanding of research concepts and the ability to reason across entire articles. In this work, we introduce LitSearch, a retrieval benchmark comprising 597 realistic literature search queries about recent ML and NLP papers. LitSearch is constructed using a combination of (1) questions generated by GPT-4 based on paragraphs containing inline citations from research papers and (2) questions manually written by authors about their recently published papers. All LitSearch questions were manually examined or edited by experts to ensure high quality. We extensively benchmark state-of-the-art retrieval models and also evaluate two LLM-based reranking pipelines. We find a significant performance gap between BM25 and state-of-the-art dense retrievers, with a 24.8% absolute difference in recall@5. The LLM-based reranking strategies further improve the best-performing dense retriever by 4.4%. Additionally, commercial search engines and research tools like Google Search perform poorly on LitSearch, lagging behind the best dense retriever by up to 32 recall points. Taken together, these results show that LitSearch is an informative new testbed for retrieval systems while catering to a real-world use case.","url_abs":"https://arxiv.org/abs/2407.18940v2","url_pdf":"https://arxiv.org/pdf/2407.18940v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"litsearch-a-retrieval-benchmark-for","repo_url":"https://github.com/princeton-nlp/litsearch","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"articles","task_name":"Articles"},{"task_slug":"reranking","task_name":"Reranking"},{"task_slug":"retrieval","task_name":"Retrieval"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt-4","method_name":"GPT-4"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2407.18940","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.18940"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/princeton-nlp/litsearch","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":7},"by_repo_kind":{"official":{"samples":7,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"d953118b232c64d9","entry":"calculate_recall","repo":"princeton-nlp/litsearch","repo_kind":"official","path":"utils/utils.py","file_url":"https://github.com/princeton-nlp/litsearch/blob/HEAD/utils/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d953118b232c64d9"}},{"code_sha256_prefix":"f9930f0c5ed5d20e","entry":"clean_response","repo":"princeton-nlp/litsearch","repo_kind":"official","path":"eval/reranking/rerank.py","file_url":"https://github.com/princeton-nlp/litsearch/blob/HEAD/eval/reranking/rerank.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f9930f0c5ed5d20e"}},{"code_sha256_prefix":"8d65f2f75a40f35a","entry":"create_prompt_messages","repo":"princeton-nlp/litsearch","repo_kind":"official","path":"eval/reranking/rerank.py","file_url":"https://github.com/princeton-nlp/litsearch/blob/HEAD/eval/reranking/rerank.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8d65f2f75a40f35a"}},{"code_sha256_prefix":"12e85238e973e6d2","entry":"get_index_name","repo":"princeton-nlp/litsearch","repo_kind":"official","path":"eval/retrieval/build_index.py","file_url":"https://github.com/princeton-nlp/litsearch/blob/HEAD/eval/retrieval/build_index.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"12e85238e973e6d2"}},{"code_sha256_prefix":"266ec98878d442f8","entry":"read_json","repo":"princeton-nlp/litsearch","repo_kind":"official","path":"utils/utils.py","file_url":"https://github.com/princeton-nlp/litsearch/blob/HEAD/utils/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"266ec98878d442f8"}},{"code_sha256_prefix":"ff8c06125817aa41","entry":"read_txt","repo":"princeton-nlp/litsearch","repo_kind":"official","path":"utils/utils.py","file_url":"https://github.com/princeton-nlp/litsearch/blob/HEAD/utils/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ff8c06125817aa41"}},{"code_sha256_prefix":"765325bbb716f8dc","entry":"remove_duplicate","repo":"princeton-nlp/litsearch","repo_kind":"official","path":"eval/reranking/rerank.py","file_url":"https://github.com/princeton-nlp/litsearch/blob/HEAD/eval/reranking/rerank.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"765325bbb716f8dc"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}