{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/s3-you-don-t-need-that-much-data-to-train-a","title":"s3: You Don't Need That Much Data to Train a Search Agent via RL","arxiv_id":"2505.14146","date":"2025-05-20","proceeding":null,"authors":["Pengcheng Jiang","Xueqiang Xu","Jiacheng Lin","Jinfeng Xiao","Zifeng Wang","Jimeng Sun","Jiawei Han"],"abstract":"Retrieval-augmented generation (RAG) systems empower large language models (LLMs) to access external knowledge during inference. Recent advances have enabled LLMs to act as search agents via reinforcement learning (RL), improving information acquisition through multi-turn interactions with retrieval engines. However, existing approaches either optimize retrieval using search-only metrics (e.g., NDCG) that ignore downstream utility or fine-tune the entire LLM to jointly reason and retrieve-entangling retrieval with generation and limiting the real search utility and compatibility with frozen or proprietary models. In this work, we propose s3, a lightweight, model-agnostic framework that decouples the searcher from the generator and trains the searcher using a Gain Beyond RAG reward: the improvement in generation accuracy over naive RAG. s3 requires only 2.4k training samples to outperform baselines trained on over 70x more data, consistently delivering stronger downstream performance across six general QA and five medical QA benchmarks.","url_abs":"https://arxiv.org/abs/2505.14146v1","url_pdf":"https://arxiv.org/pdf/2505.14146v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"s3-you-don-t-need-that-much-data-to-train-a","repo_url":"https://github.com/pat-jj/s3","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"rag","task_name":"RAG"},{"task_slug":"reinforcement-learning-1","task_name":"Reinforcement Learning (RL)"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"retrieval-augmented-generation","task_name":"Retrieval-augmented Generation"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bart","method_name":"BART"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"rag","method_name":"RAG"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2505.14146","atlas_url":"https://app.syntology.ai/?focus=2505.14146","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2505.14146"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/pat-jj/s3","reach":null}],"summary":{"ran":3,"ran_draft_wrong":3,"ran_fixture":1,"unverified":2},"by_repo_kind":{"official":{"samples":9,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"3ddd2f19ce79889e","entry":"DataProto","repo":"pat-jj/s3","repo_kind":"official","path":"s3/llm_agent/generation_s3.py","file_url":"https://github.com/pat-jj/s3/blob/HEAD/s3/llm_agent/generation_s3.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3ddd2f19ce79889e"}},{"code_sha256_prefix":"c184c5af900dddb0","entry":"DataProtoItem","repo":"pat-jj/s3","repo_kind":"official","path":"s3/llm_agent/generation_s3.py","file_url":"https://github.com/pat-jj/s3/blob/HEAD/s3/llm_agent/generation_s3.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c184c5af900dddb0"}},{"code_sha256_prefix":"a93d6a0502295f81","entry":"GenerationConfig","repo":"pat-jj/s3","repo_kind":"official","path":"s3/llm_agent/generation_s3.py","file_url":"https://github.com/pat-jj/s3/blob/HEAD/s3/llm_agent/generation_s3.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a93d6a0502295f81"}},{"code_sha256_prefix":"8709d15899e3d6a2","entry":"list_of_dict_to_dict_of_list","repo":"pat-jj/s3","repo_kind":"official","path":"s3/llm_agent/generation_s3.py","file_url":"https://github.com/pat-jj/s3/blob/HEAD/s3/llm_agent/generation_s3.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8709d15899e3d6a2"}},{"code_sha256_prefix":"0ee139479f9aed81","entry":"union_numpy_dict","repo":"pat-jj/s3","repo_kind":"official","path":"s3/llm_agent/generation_s3.py","file_url":"https://github.com/pat-jj/s3/blob/HEAD/s3/llm_agent/generation_s3.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0ee139479f9aed81"}},{"code_sha256_prefix":"21331a58f93375e4","entry":"union_tensor_dict","repo":"pat-jj/s3","repo_kind":"official","path":"s3/llm_agent/generation_s3.py","file_url":"https://github.com/pat-jj/s3/blob/HEAD/s3/llm_agent/generation_s3.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"DEP_MISSING","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"21331a58f93375e4"}},{"code_sha256_prefix":"084b70223fc68422","entry":"union_two_dict","repo":"pat-jj/s3","repo_kind":"official","path":"s3/llm_agent/generation_s3.py","file_url":"https://github.com/pat-jj/s3/blob/HEAD/s3/llm_agent/generation_s3.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"084b70223fc68422"}},{"code_sha256_prefix":"a42a0e447f1a6c68","entry":"LLMGenerationManager","repo":"pat-jj/s3","repo_kind":"official","path":"s3/llm_agent/generation_s3.py","file_url":"https://github.com/pat-jj/s3/blob/HEAD/s3/llm_agent/generation_s3.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a42a0e447f1a6c68"}},{"code_sha256_prefix":"90f6865eb340d338","entry":"collate_fn","repo":"pat-jj/s3","repo_kind":"official","path":"s3/llm_agent/generation_s3.py","file_url":"https://github.com/pat-jj/s3/blob/HEAD/s3/llm_agent/generation_s3.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"90f6865eb340d338"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}