{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/fine-tuning-llama-for-multi-stage-text","title":"Fine-Tuning LLaMA for Multi-Stage Text Retrieval","arxiv_id":"2310.08319","date":"2023-10-12","proceeding":null,"authors":["Xueguang Ma","Liang Wang","Nan Yang","Furu Wei","Jimmy Lin"],"abstract":"The effectiveness of multi-stage text retrieval has been solidly demonstrated since before the era of pre-trained language models. However, most existing studies utilize models that predate recent advances in large language models (LLMs). This study seeks to explore potential improvements that state-of-the-art LLMs can bring. We conduct a comprehensive study, fine-tuning the latest LLaMA model both as a dense retriever (RepLLaMA) and as a pointwise reranker (RankLLaMA) for both passage retrieval and document retrieval using the MS MARCO datasets. Our findings demonstrate that the effectiveness of large language models indeed surpasses that of smaller models. Additionally, since LLMs can inherently handle longer contexts, they can represent entire documents holistically, obviating the need for traditional segmenting and pooling strategies. Furthermore, evaluations on BEIR demonstrate that our RepLLaMA-RankLLaMA pipeline exhibits strong zero-shot effectiveness. Model checkpoints from this study are available on HuggingFace.","url_abs":"https://arxiv.org/abs/2310.08319v1","url_pdf":"https://arxiv.org/pdf/2310.08319v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"fine-tuning-llama-for-multi-stage-text","repo_url":"https://github.com/emory-irlab/pyterrier_genrank","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"fine-tuning-llama-for-multi-stage-text","repo_url":"https://github.com/texttron/tevatron/tree/main/examples/repllama","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"jax","reach":null}],"tasks":[{"task_slug":"passage-retrieval","task_name":"Passage Retrieval"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"text-retrieval","task_name":"Text Retrieval"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2310.08319","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2310.08319"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/emory-irlab/pyterrier_genrank","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/texttron/tevatron/tree/main/examples/repllama","reach":null}],"summary":{"ran":8,"unverified":1},"by_repo_kind":{"listed":{"samples":9,"ran":8,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"58ca9eadffe169a1","entry":"beir_data_path","repo":"texttron/tevatron","repo_kind":"listed","path":"src/tevatron/eval/benchmarks.py","file_url":"https://github.com/texttron/tevatron/blob/HEAD/src/tevatron/eval/benchmarks.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"58ca9eadffe169a1"}},{"code_sha256_prefix":"da58dea30fe4d922","entry":"beir_split_for","repo":"texttron/tevatron","repo_kind":"listed","path":"src/tevatron/eval/benchmarks.py","file_url":"https://github.com/texttron/tevatron/blob/HEAD/src/tevatron/eval/benchmarks.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"da58dea30fe4d922"}},{"code_sha256_prefix":"f0cd0ee4f1a7e335","entry":"beir_url_and_subdir","repo":"texttron/tevatron","repo_kind":"listed","path":"src/tevatron/eval/benchmarks.py","file_url":"https://github.com/texttron/tevatron/blob/HEAD/src/tevatron/eval/benchmarks.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f0cd0ee4f1a7e335"}},{"code_sha256_prefix":"cd7cef6e9418ab5d","entry":"last_token_pool","repo":"texttron/tevatron","repo_kind":"listed","path":"src/tevatron/megatron/engine.py","file_url":"https://github.com/texttron/tevatron/blob/HEAD/src/tevatron/megatron/engine.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cd7cef6e9418ab5d"}},{"code_sha256_prefix":"4b8a981075a9c13c","entry":"read_ranklist","repo":"texttron/tevatron","repo_kind":"listed","path":"src/tevatron/eval/metrics.py","file_url":"https://github.com/texttron/tevatron/blob/HEAD/src/tevatron/eval/metrics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4b8a981075a9c13c"}},{"code_sha256_prefix":"b118264a1ed82791","entry":"render","repo":"texttron/tevatron","repo_kind":"listed","path":"src/tevatron/eval/summary.py","file_url":"https://github.com/texttron/tevatron/blob/HEAD/src/tevatron/eval/summary.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b118264a1ed82791"}},{"code_sha256_prefix":"81c36957e2034400","entry":"reranker_loss","repo":"texttron/tevatron","repo_kind":"listed","path":"src/tevatron/megatron/loss.py","file_url":"https://github.com/texttron/tevatron/blob/HEAD/src/tevatron/megatron/loss.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"81c36957e2034400"}},{"code_sha256_prefix":"5eb05d1d51ec1fa5","entry":"score","repo":"texttron/tevatron","repo_kind":"listed","path":"src/tevatron/eval/metrics.py","file_url":"https://github.com/texttron/tevatron/blob/HEAD/src/tevatron/eval/metrics.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5eb05d1d51ec1fa5"}},{"code_sha256_prefix":"f560d59e5a1e1ace","entry":"expand_lora_target_groups","repo":"texttron/tevatron","repo_kind":"listed","path":"src/tevatron/megatron/config.py","file_url":"https://github.com/texttron/tevatron/blob/HEAD/src/tevatron/megatron/config.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f560d59e5a1e1ace"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}