{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/lightretriever-a-llm-based-hybrid-retrieval","title":"LightRetriever: A LLM-based Hybrid Retrieval Architecture with 1000x Faster Query Inference","arxiv_id":"2505.12260","date":"2025-05-18","proceeding":null,"authors":["Guangyuan Ma","Yongliang Ma","Xuanrui Gou","Zhenpeng Su","Ming Zhou","Songlin Hu"],"abstract":"Large Language Models (LLMs)-based hybrid retrieval uses LLMs to encode queries and documents into low-dimensional dense or high-dimensional sparse vectors. It retrieves documents relevant to search queries based on vector similarities. Documents are pre-encoded offline, while queries arrive in real-time, necessitating an efficient online query encoder. Although LLMs significantly enhance retrieval capabilities, serving deeply parameterized LLMs slows down query inference throughput and increases demands for online deployment resources. In this paper, we propose LightRetriever, a novel LLM-based hybrid retriever with extremely lightweight query encoders. Our method retains a full-sized LLM for document encoding, but reduces the workload of query encoding to no more than an embedding lookup. Compared to serving a full-sized LLM on an H800 GPU, our approach achieves over a 1000x speedup for query inference with GPU acceleration, and even a 20x speedup without GPU. Experiments on large-scale retrieval benchmarks demonstrate that our method generalizes well across diverse retrieval tasks, retaining an average of 95% full-sized performance.","url_abs":"https://arxiv.org/abs/2505.12260v1","url_pdf":"https://arxiv.org/pdf/2505.12260v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"lightretriever-a-llm-based-hybrid-retrieval","repo_url":"https://github.com/caskcsg/lightretriever","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":null,"task_name":"GPU"},{"task_slug":"retrieval","task_name":"Retrieval"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2505.12260","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.12260"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/caskcsg/lightretriever","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran_draft_wrong":1,"ran":1,"unverified":9},"by_repo_kind":{"official":{"samples":11,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"43e79e8e0904d5d8","entry":"dcg_score","repo":"caskcsg/lightretriever","repo_kind":"official","path":"src/lightretriever/finetune/metrics.py","file_url":"https://github.com/caskcsg/lightretriever/blob/HEAD/src/lightretriever/finetune/metrics.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"43e79e8e0904d5d8"}},{"code_sha256_prefix":"aaa4cece9f41ea15","entry":"ndcg_score","repo":"caskcsg/lightretriever","repo_kind":"official","path":"src/lightretriever/finetune/metrics.py","file_url":"https://github.com/caskcsg/lightretriever/blob/HEAD/src/lightretriever/finetune/metrics.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"aaa4cece9f41ea15"}},{"code_sha256_prefix":"97574ff51cffea4e","entry":"find_closest_power_of_2","repo":"caskcsg/lightretriever","repo_kind":"official","path":"src/lightretriever/trainer.py","file_url":"https://github.com/caskcsg/lightretriever/blob/HEAD/src/lightretriever/trainer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"97574ff51cffea4e"}},{"code_sha256_prefix":"d091bfa8194ca89f","entry":"get_cosine_schedule_with_warmup_minlr","repo":"caskcsg/lightretriever","repo_kind":"official","path":"src/lightretriever/scheduler.py","file_url":"https://github.com/caskcsg/lightretriever/blob/HEAD/src/lightretriever/scheduler.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d091bfa8194ca89f"}},{"code_sha256_prefix":"9aaa2ddc83faa93d","entry":"get_linear_schedule_with_warmup_minlr","repo":"caskcsg/lightretriever","repo_kind":"official","path":"src/lightretriever/scheduler.py","file_url":"https://github.com/caskcsg/lightretriever/blob/HEAD/src/lightretriever/scheduler.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9aaa2ddc83faa93d"}},{"code_sha256_prefix":"8795573bc2983705","entry":"get_lm_head","repo":"caskcsg/lightretriever","repo_kind":"official","path":"src/lightretriever/finetune/modeling_hybrid.py","file_url":"https://github.com/caskcsg/lightretriever/blob/HEAD/src/lightretriever/finetune/modeling_hybrid.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8795573bc2983705"}},{"code_sha256_prefix":"7ec8ad33274123d1","entry":"get_reg_weight_scaling_factor","repo":"caskcsg/lightretriever","repo_kind":"official","path":"src/lightretriever/scheduler.py","file_url":"https://github.com/caskcsg/lightretriever/blob/HEAD/src/lightretriever/scheduler.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7ec8ad33274123d1"}},{"code_sha256_prefix":"cd3a8d8e0d35ed6b","entry":"mean_eos_pooling","repo":"caskcsg/lightretriever","repo_kind":"official","path":"src/lightretriever/finetune/dense_pooling.py","file_url":"https://github.com/caskcsg/lightretriever/blob/HEAD/src/lightretriever/finetune/dense_pooling.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"cd3a8d8e0d35ed6b"}},{"code_sha256_prefix":"c1933bf46ff5c9b6","entry":"mrr_score","repo":"caskcsg/lightretriever","repo_kind":"official","path":"src/lightretriever/finetune/metrics.py","file_url":"https://github.com/caskcsg/lightretriever/blob/HEAD/src/lightretriever/finetune/metrics.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c1933bf46ff5c9b6"}},{"code_sha256_prefix":"028d8766e22f9773","entry":"pooling","repo":"caskcsg/lightretriever","repo_kind":"official","path":"src/lightretriever/finetune/dense_pooling.py","file_url":"https://github.com/caskcsg/lightretriever/blob/HEAD/src/lightretriever/finetune/dense_pooling.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"028d8766e22f9773"}},{"code_sha256_prefix":"d14a93abddd46c17","entry":"rewrite_logs","repo":"caskcsg/lightretriever","repo_kind":"official","path":"src/lightretriever/trainer.py","file_url":"https://github.com/caskcsg/lightretriever/blob/HEAD/src/lightretriever/trainer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d14a93abddd46c17"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}