{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/ragged-towards-informed-design-of-retrieval","title":"RAGGED: Towards Informed Design of Retrieval Augmented Generation Systems","arxiv_id":"2403.09040","date":"2024-03-14","proceeding":null,"authors":["Jennifer Hsia","Afreen Shaikh","Zhiruo Wang","Graham Neubig"],"abstract":"Retrieval-augmented generation (RAG) can significantly improve the performance of language models (LMs) by providing additional context for tasks such as document-based question answering (DBQA). However, the effectiveness of RAG is highly dependent on its configuration. To systematically find the optimal configuration, we introduce RAGGED, a framework for analyzing RAG configurations across various DBQA tasks. Using the framework, we discover distinct LM behaviors in response to varying context quantities, context qualities, and retrievers. For instance, while some models are robust to noisy contexts, monotonically performing better with more contexts, others are more noise-sensitive and can effectively use only a few contexts before declining in performance. This framework also provides a deeper analysis of these differences by evaluating the LMs' sensitivity to signal and noise under specific context quality conditions. Using RAGGED, researchers and practitioners can derive actionable insights about how to optimally configure their RAG systems for their specific question-answering tasks.","url_abs":"https://arxiv.org/abs/2403.09040v2","url_pdf":"https://arxiv.org/pdf/2403.09040v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"ragged-towards-informed-design-of-retrieval","repo_url":"https://github.com/neulab/ragged","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"decoder","task_name":"Decoder"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"rag","task_name":"RAG"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"retrieval-augmented-generation","task_name":"Retrieval-augmented Generation"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bart","method_name":"BART"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"rag","method_name":"RAG"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2403.09040","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2403.09040"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/neulab/ragged","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran":12,"ran_draft_wrong":1,"unverified":1},"by_repo_kind":{"official":{"samples":14,"ran":13,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"f220897243bce737","entry":"convert_reader_results_to_zeno","repo":"neulab/ragged","repo_kind":"official","path":"analysis_framework/compile_results.py","file_url":"https://github.com/neulab/ragged/blob/HEAD/analysis_framework/compile_results.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f220897243bce737"}},{"code_sha256_prefix":"ca677f8c2b43120d","entry":"count_jsonl","repo":"neulab/ragged","repo_kind":"official","path":"retriever/evaluate_retriever.py","file_url":"https://github.com/neulab/ragged/blob/HEAD/retriever/evaluate_retriever.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ca677f8c2b43120d"}},{"code_sha256_prefix":"289d7efd299b90d4","entry":"create_prompt","repo":"neulab/ragged","repo_kind":"official","path":"reader/utils.py","file_url":"https://github.com/neulab/ragged/blob/HEAD/reader/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"289d7efd299b90d4"}},{"code_sha256_prefix":"1b1fbbd9dac2072c","entry":"get_precision","repo":"neulab/ragged","repo_kind":"official","path":"analysis_framework/zeno_upload.py","file_url":"https://github.com/neulab/ragged/blob/HEAD/analysis_framework/zeno_upload.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1b1fbbd9dac2072c"}},{"code_sha256_prefix":"d05429bcacc8eccc","entry":"get_precision","repo":"neulab/ragged","repo_kind":"official","path":"retriever/evaluate_retriever.py","file_url":"https://github.com/neulab/ragged/blob/HEAD/retriever/evaluate_retriever.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d05429bcacc8eccc"}},{"code_sha256_prefix":"19d64b24f5ce24e3","entry":"get_recall","repo":"neulab/ragged","repo_kind":"official","path":"analysis_framework/zeno_upload.py","file_url":"https://github.com/neulab/ragged/blob/HEAD/analysis_framework/zeno_upload.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"19d64b24f5ce24e3"}},{"code_sha256_prefix":"a072fc24dd6a64f4","entry":"get_recall","repo":"neulab/ragged","repo_kind":"official","path":"retriever/evaluate_retriever.py","file_url":"https://github.com/neulab/ragged/blob/HEAD/retriever/evaluate_retriever.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a072fc24dd6a64f4"}},{"code_sha256_prefix":"507cd94cf077b40b","entry":"is_potential_number","repo":"neulab/ragged","repo_kind":"official","path":"reader/evaluate_top_k.py","file_url":"https://github.com/neulab/ragged/blob/HEAD/reader/evaluate_top_k.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"507cd94cf077b40b"}},{"code_sha256_prefix":"130b876997375207","entry":"load_json","repo":"neulab/ragged","repo_kind":"official","path":"file_utils.py","file_url":"https://github.com/neulab/ragged/blob/HEAD/file_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"130b876997375207"}},{"code_sha256_prefix":"ca1dbea1a9af7003","entry":"load_jsonl","repo":"neulab/ragged","repo_kind":"official","path":"file_utils.py","file_url":"https://github.com/neulab/ragged/blob/HEAD/file_utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ca1dbea1a9af7003"}},{"code_sha256_prefix":"e7e75981cb464788","entry":"normalize_answer","repo":"neulab/ragged","repo_kind":"official","path":"reader/evaluate_top_k.py","file_url":"https://github.com/neulab/ragged/blob/HEAD/reader/evaluate_top_k.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e7e75981cb464788"}},{"code_sha256_prefix":"d0896f977a782cd6","entry":"post_process_answers","repo":"neulab/ragged","repo_kind":"official","path":"reader/utils.py","file_url":"https://github.com/neulab/ragged/blob/HEAD/reader/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d0896f977a782cd6"}},{"code_sha256_prefix":"f6e6c7c81db7d3e1","entry":"truncate_prompt","repo":"neulab/ragged","repo_kind":"official","path":"reader/utils.py","file_url":"https://github.com/neulab/ragged/blob/HEAD/reader/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f6e6c7c81db7d3e1"}},{"code_sha256_prefix":"6e7ed506217fac36","entry":"get_tokenizer","repo":"neulab/ragged","repo_kind":"official","path":"utils.py","file_url":"https://github.com/neulab/ragged/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6e7ed506217fac36"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}