{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/self-retrieval-building-an-information","title":"Self-Retrieval: End-to-End Information Retrieval with One Large Language Model","arxiv_id":"2403.00801","date":"2024-02-23","proceeding":null,"authors":["Qiaoyu Tang","Jiawei Chen","Zhuoqun Li","Bowen Yu","Yaojie Lu","Cheng Fu","Haiyang Yu","Hongyu Lin","Fei Huang","Ben He","Xianpei Han","Le Sun","Yongbin Li"],"abstract":"The rise of large language models (LLMs) has significantly transformed both the construction and application of information retrieval (IR) systems. However, current interactions between IR systems and LLMs remain limited, with LLMs merely serving as part of components within IR systems, and IR systems being constructed independently of LLMs. This separated architecture restricts knowledge sharing and deep collaboration between them. In this paper, we introduce Self-Retrieval, a novel end-to-end LLM-driven information retrieval architecture. Self-Retrieval unifies all essential IR functions within a single LLM, leveraging the inherent capabilities of LLMs throughout the IR process. Specifically, Self-Retrieval internalizes the retrieval corpus through self-supervised learning, transforms the retrieval process into sequential passage generation, and performs relevance assessment for reranking. Experimental results demonstrate that Self-Retrieval not only outperforms existing retrieval approaches by a significant margin, but also substantially enhances the performance of LLM-driven downstream applications like retrieval-augmented generation.","url_abs":"https://arxiv.org/abs/2403.00801v2","url_pdf":"https://arxiv.org/pdf/2403.00801v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"self-retrieval-building-an-information","repo_url":"https://github.com/icip-cas/selfretrieval","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null},{"paper_slug":"self-retrieval-building-an-information","repo_url":"https://github.com/tangqiaoyu/selfretrieval","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"information-retrieval","task_name":"Information Retrieval"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"large-language-model","task_name":"Large Language Model"},{"task_slug":"reranking","task_name":"Reranking"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"retrieval-augmented-generation","task_name":"Retrieval-augmented Generation"},{"task_slug":"self-supervised-learning","task_name":"Self-Supervised Learning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2403.00801","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.00801"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/icip-cas/selfretrieval","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/tangqiaoyu/selfretrieval","reach":null}],"summary":{"ran_draft_wrong":1,"ran_honours":1,"unverified":2},"by_repo_kind":{"official":{"samples":4,"ran":2,"repositories":2}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":4,"samples":[{"code_sha256_prefix":"fab65e2bedd8dec0","entry":"get_final_results_from_prefix","repo":"icip-cas/selfretrieval","repo_kind":"official","path":"evaluation/predict.py","file_url":"https://github.com/icip-cas/selfretrieval/blob/HEAD/evaluation/predict.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fab65e2bedd8dec0"}},{"code_sha256_prefix":"d208bd8d5e5bdb52","entry":"self_assessment","repo":"icip-cas/selfretrieval","repo_kind":"official","path":"evaluation/predict.py","file_url":"https://github.com/icip-cas/selfretrieval/blob/HEAD/evaluation/predict.py","link_basis":"first_harvest_node","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d208bd8d5e5bdb52"}},{"code_sha256_prefix":"0cfdc6cb7b4cd04e","entry":"Retriever","repo":"tangqiaoyu/selfretrieval","repo_kind":"official","path":"evaluation/predict.py","file_url":"https://github.com/tangqiaoyu/selfretrieval/blob/HEAD/evaluation/predict.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0cfdc6cb7b4cd04e"}},{"code_sha256_prefix":"55a6e80874eda228","entry":"constrain_func","repo":"icip-cas/selfretrieval","repo_kind":"official","path":"evaluation/predict.py","file_url":"https://github.com/icip-cas/selfretrieval/blob/HEAD/evaluation/predict.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"55a6e80874eda228"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}