{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/ragnarok-a-reusable-rag-framework-and","title":"Ragnarök: A Reusable RAG Framework and Baselines for TREC 2024 Retrieval-Augmented Generation Track","arxiv_id":"2406.16828","date":"2024-06-24","proceeding":null,"authors":["Ronak Pradeep","Nandan Thakur","Sahel Sharifymoghaddam","Eric Zhang","Ryan Nguyen","Daniel Campos","Nick Craswell","Jimmy Lin"],"abstract":"Did you try out the new Bing Search? Or maybe you fiddled around with Google AI~Overviews? These might sound familiar because the modern-day search stack has recently evolved to include retrieval-augmented generation (RAG) systems. They allow searching and incorporating real-time data into large language models (LLMs) to provide a well-informed, attributed, concise summary in contrast to the traditional search paradigm that relies on displaying a ranked list of documents. Therefore, given these recent advancements, it is crucial to have an arena to build, test, visualize, and systematically evaluate RAG-based search systems. With this in mind, we propose the TREC 2024 RAG Track to foster innovation in evaluating RAG systems. In our work, we lay out the steps we've made towards making this track a reality -- we describe the details of our reusable framework, Ragnar\\\"ok, explain the curation of the new MS MARCO V2.1 collection choice, release the development topics for the track, and standardize the I/O definitions which assist the end user. Next, using Ragnar\\\"ok, we identify and provide key industrial baselines such as OpenAI's GPT-4o or Cohere's Command R+. Further, we introduce a web-based user interface for an interactive arena allowing benchmarking pairwise RAG systems by crowdsourcing. We open-source our Ragnar\\\"ok framework and baselines to achieve a unified standard for future RAG systems.","url_abs":"https://arxiv.org/abs/2406.16828v1","url_pdf":"https://arxiv.org/pdf/2406.16828v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"ragnarok-a-reusable-rag-framework-and","repo_url":"https://github.com/castorini/ragnarok","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"ragnarok-a-reusable-rag-framework-and","repo_url":"https://github.com/castorini/nuggetizer","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"benchmarking","task_name":"Benchmarking"},{"task_slug":"rag","task_name":"RAG"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"retrieval-augmented-generation","task_name":"Retrieval-augmented Generation"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bart","method_name":"BART"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"rag","method_name":"RAG"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2406.16828","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2406.16828"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/castorini/nuggetizer","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/castorini/ragnarok","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":9,"unverified":14},"by_repo_kind":{"official":{"samples":12,"ran":9,"repositories":1},"listed":{"samples":11,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"d76974dd615c64e1","entry":"compute_elo","repo":"castorini/ragnarok","repo_kind":"official","path":"src/ragnarok/api/elo.py","file_url":"https://github.com/castorini/ragnarok/blob/HEAD/src/ragnarok/api/elo.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d76974dd615c64e1"}},{"code_sha256_prefix":"dd91db53416878f5","entry":"make_data_artifact","repo":"castorini/ragnarok","repo_kind":"official","path":"src/ragnarok/cli/adapters.py","file_url":"https://github.com/castorini/ragnarok/blob/HEAD/src/ragnarok/cli/adapters.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"dd91db53416878f5"}},{"code_sha256_prefix":"7544006c2fb2eb8d","entry":"make_file_artifact","repo":"castorini/ragnarok","repo_kind":"official","path":"src/ragnarok/cli/adapters.py","file_url":"https://github.com/castorini/ragnarok/blob/HEAD/src/ragnarok/cli/adapters.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7544006c2fb2eb8d"}},{"code_sha256_prefix":"4a89617557cd8d60","entry":"prepare_output_path","repo":"castorini/ragnarok","repo_kind":"official","path":"src/ragnarok/cli/dispatch.py","file_url":"https://github.com/castorini/ragnarok/blob/HEAD/src/ragnarok/cli/dispatch.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4a89617557cd8d60"}},{"code_sha256_prefix":"f3bfd9301b14fe26","entry":"read_direct_payload","repo":"castorini/ragnarok","repo_kind":"official","path":"src/ragnarok/cli/dispatch.py","file_url":"https://github.com/castorini/ragnarok/blob/HEAD/src/ragnarok/cli/dispatch.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f3bfd9301b14fe26"}},{"code_sha256_prefix":"e653a76dcbab5bab","entry":"read_results_from_file","repo":"castorini/ragnarok","repo_kind":"official","path":"src/ragnarok/data.py","file_url":"https://github.com/castorini/ragnarok/blob/HEAD/src/ragnarok/data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e653a76dcbab5bab"}},{"code_sha256_prefix":"33cd494e3b5331cf","entry":"resolve_write_policy","repo":"castorini/ragnarok","repo_kind":"official","path":"src/ragnarok/cli/dispatch.py","file_url":"https://github.com/castorini/ragnarok/blob/HEAD/src/ragnarok/cli/dispatch.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"33cd494e3b5331cf"}},{"code_sha256_prefix":"4a11e6b65475e860","entry":"result_to_dict","repo":"castorini/ragnarok","repo_kind":"official","path":"src/ragnarok/data.py","file_url":"https://github.com/castorini/ragnarok/blob/HEAD/src/ragnarok/data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4a11e6b65475e860"}},{"code_sha256_prefix":"c53a0ca89658e42c","entry":"wants_json","repo":"castorini/ragnarok","repo_kind":"official","path":"src/ragnarok/cli/errors.py","file_url":"https://github.com/castorini/ragnarok/blob/HEAD/src/ragnarok/cli/errors.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c53a0ca89658e42c"}},{"code_sha256_prefix":"b072bca5935fc26d","entry":"build_assign_nuggetizer_kwargs","repo":"castorini/nuggetizer","repo_kind":"listed","path":"src/nuggetizer/cli/execution.py","file_url":"https://github.com/castorini/nuggetizer/blob/HEAD/src/nuggetizer/cli/execution.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b072bca5935fc26d"}},{"code_sha256_prefix":"c775437a8c8fe07d","entry":"build_create_nuggetizer_kwargs","repo":"castorini/nuggetizer","repo_kind":"listed","path":"src/nuggetizer/cli/execution.py","file_url":"https://github.com/castorini/nuggetizer/blob/HEAD/src/nuggetizer/cli/execution.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c775437a8c8fe07d"}},{"code_sha256_prefix":"53b36b0cccd27107","entry":"collect_nonempty_reasoning_traces","repo":"castorini/nuggetizer","repo_kind":"listed","path":"src/nuggetizer/cli/adapters.py","file_url":"https://github.com/castorini/nuggetizer/blob/HEAD/src/nuggetizer/cli/adapters.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"53b36b0cccd27107"}},{"code_sha256_prefix":"258e9e13fcbf5b84","entry":"detect_command","repo":"castorini/ragnarok","repo_kind":"official","path":"src/ragnarok/cli/errors.py","file_url":"https://github.com/castorini/ragnarok/blob/HEAD/src/ragnarok/cli/errors.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"258e9e13fcbf5b84"}},{"code_sha256_prefix":"145f2c77b2912062","entry":"detect_command","repo":"castorini/nuggetizer","repo_kind":"listed","path":"src/nuggetizer/cli/main_support.py","file_url":"https://github.com/castorini/nuggetizer/blob/HEAD/src/nuggetizer/cli/main_support.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"145f2c77b2912062"}},{"code_sha256_prefix":"7f2886141fa690bf","entry":"get_processed_values","repo":"castorini/nuggetizer","repo_kind":"listed","path":"src/nuggetizer/cli/io.py","file_url":"https://github.com/castorini/nuggetizer/blob/HEAD/src/nuggetizer/cli/io.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7f2886141fa690bf"}},{"code_sha256_prefix":"8fa7a665d31a83dc","entry":"get_run_id","repo":"castorini/nuggetizer","repo_kind":"listed","path":"src/nuggetizer/cli/io.py","file_url":"https://github.com/castorini/nuggetizer/blob/HEAD/src/nuggetizer/cli/io.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8fa7a665d31a83dc"}},{"code_sha256_prefix":"719779e74d1ac660","entry":"read_jsonl","repo":"castorini/nuggetizer","repo_kind":"listed","path":"src/nuggetizer/cli/io.py","file_url":"https://github.com/castorini/nuggetizer/blob/HEAD/src/nuggetizer/cli/io.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"719779e74d1ac660"}},{"code_sha256_prefix":"b74163bc3735c7ee","entry":"setup_logging","repo":"castorini/nuggetizer","repo_kind":"listed","path":"src/nuggetizer/cli/logging_utils.py","file_url":"https://github.com/castorini/nuggetizer/blob/HEAD/src/nuggetizer/cli/logging_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"b74163bc3735c7ee"}},{"code_sha256_prefix":"074a68a9b4bcba9f","entry":"unwrap_direct_create_payload","repo":"castorini/nuggetizer","repo_kind":"listed","path":"src/nuggetizer/cli/normalize.py","file_url":"https://github.com/castorini/nuggetizer/blob/HEAD/src/nuggetizer/cli/normalize.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"074a68a9b4bcba9f"}},{"code_sha256_prefix":"74873c74cd75260c","entry":"unwrap_generation_record","repo":"castorini/nuggetizer","repo_kind":"listed","path":"src/nuggetizer/cli/normalize.py","file_url":"https://github.com/castorini/nuggetizer/blob/HEAD/src/nuggetizer/cli/normalize.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"74873c74cd75260c"}},{"code_sha256_prefix":"fd5baa7869c054bc","entry":"unwrap_nugget_record","repo":"castorini/nuggetizer","repo_kind":"listed","path":"src/nuggetizer/cli/normalize.py","file_url":"https://github.com/castorini/nuggetizer/blob/HEAD/src/nuggetizer/cli/normalize.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fd5baa7869c054bc"}},{"code_sha256_prefix":"2016f3275f584d17","entry":"validate_generate_batch_file","repo":"castorini/ragnarok","repo_kind":"official","path":"src/ragnarok/cli/introspection.py","file_url":"https://github.com/castorini/ragnarok/blob/HEAD/src/ragnarok/cli/introspection.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2016f3275f584d17"}},{"code_sha256_prefix":"03080e064ef748c8","entry":"validate_generate_payload","repo":"castorini/ragnarok","repo_kind":"official","path":"src/ragnarok/cli/introspection.py","file_url":"https://github.com/castorini/ragnarok/blob/HEAD/src/ragnarok/cli/introspection.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"03080e064ef748c8"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}