{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/arxiv-2601-16503","title":"MRAG: Benchmarking Retrieval-Augmented Generation for Bio-medicine","arxiv_id":"2601.16503","date":"2026-01-23","proceeding":null,"authors":["Liz Li","Wei Zhu"],"abstract":"While Retrieval-Augmented Generation (RAG) has been swiftly adopted in scientific and clinical QA systems, a comprehensive evaluation benchmark in the medical domain is lacking. To address this gap, we introduce the Medical Retrieval-Augmented Generation (MRAG) benchmark, covering various tasks in English and Chinese languages, and building a corpus with Wikipedia and Pubmed. Additionally, we develop the MRAG-Toolkit, facilitating systematic exploration of different RAG components. Our experiments reveal that: (a) RAG enhances LLM reliability across MRAG tasks. (b) the performance of RAG systems is influenced by retrieval approaches, model sizes, and prompting strategies. (c) While RAG improves usefulness and reasoning quality, LLM responses may become slightly less readable for long-form questions. We will release the MRAG-Bench's dataset and toolkit with CCBY-4.0 license upon acceptance, to facilitate applications from both academia and industry.","url_abs":"https://arxiv.org/abs/2601.16503","url_pdf":"https://arxiv.org/pdf/2601.16503","source":{"archive":null,"snapshot":"2025-07-28","note":"not in the Papers with Code archive (frozen at the snapshot)","row_kind":"graph","title_abstract_authors_date":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)"},"code_links":[],"tasks":[],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2601.16503","atlas_url":"https://app.syntology.ai/?focus=2601.16503","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2601.16503"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/hendrycks/test","reach":null},{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/facebookresearch/faiss","reach":null},{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/jind11/MedQA","reach":null},{"mentioned_in_github":null,"is_official":null,"provenance":"deterministic:regex_extraction","mentioned_in_paper":null,"url":"https://github.com/isegura/DDICorpus","reach":{"status":"ok"}}],"summary":{"ran_fixture":1,"ran_draft_wrong":2,"ran":1,"unverified":5},"by_repo_kind":{"found_in_text":{"samples":9,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"19f01570b0a18e2b","entry":"softmax","repo":"hendrycks/test","repo_kind":"found_in_text","path":"evaluate.py","file_url":"https://github.com/hendrycks/test/blob/HEAD/evaluate.py","link_basis":"plan_row","language":"python","status":"ran_fixture","verification_level":2,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"19f01570b0a18e2b"}},{"code_sha256_prefix":"cd763eaf1ac287e7","entry":"format_example","repo":"hendrycks/test","repo_kind":"found_in_text","path":"evaluate.py","file_url":"https://github.com/hendrycks/test/blob/HEAD/evaluate.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cd763eaf1ac287e7"}},{"code_sha256_prefix":"6ab745408cb8648b","entry":"format_subject","repo":"hendrycks/test","repo_kind":"found_in_text","path":"evaluate.py","file_url":"https://github.com/hendrycks/test/blob/HEAD/evaluate.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6ab745408cb8648b"}},{"code_sha256_prefix":"3383488ea82b6c15","entry":"gen_prompt","repo":"hendrycks/test","repo_kind":"found_in_text","path":"evaluate_flan.py","file_url":"https://github.com/hendrycks/test/blob/HEAD/evaluate_flan.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3383488ea82b6c15"}},{"code_sha256_prefix":"432838d3c5f2ab75","entry":"get_adaptive_bins","repo":"hendrycks/test","repo_kind":"found_in_text","path":"calib_tools.py","file_url":"https://github.com/hendrycks/test/blob/HEAD/calib_tools.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"432838d3c5f2ab75"}},{"code_sha256_prefix":"3c57d0ee2854879f","entry":"get_encoder","repo":"hendrycks/test","repo_kind":"found_in_text","path":"crop.py","file_url":"https://github.com/hendrycks/test/blob/HEAD/crop.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3c57d0ee2854879f"}},{"code_sha256_prefix":"4f76173dfe4b0c11","entry":"get_pairs","repo":"hendrycks/test","repo_kind":"found_in_text","path":"crop.py","file_url":"https://github.com/hendrycks/test/blob/HEAD/crop.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4f76173dfe4b0c11"}},{"code_sha256_prefix":"0af29c87aa4bd6a8","entry":"mean","repo":"hendrycks/test","repo_kind":"found_in_text","path":"calib_tools.py","file_url":"https://github.com/hendrycks/test/blob/HEAD/calib_tools.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0af29c87aa4bd6a8"}},{"code_sha256_prefix":"77933604fbca56e7","entry":"one_hot_encode","repo":"hendrycks/test","repo_kind":"found_in_text","path":"calib_tools.py","file_url":"https://github.com/hendrycks/test/blob/HEAD/calib_tools.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"77933604fbca56e7"}}]},"arxiv_metadata":{"licence":"arXiv metadata, CC0 1.0 (https://info.arxiv.org/help/license)","fields":["title","abstract","authors","date"],"primary_category":"cs.CL","source":"arxiv_2026.jsonl"},"syntology_extracted_results":null}