{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/ares-an-automated-evaluation-framework-for","title":"ARES: An Automated Evaluation Framework for Retrieval-Augmented Generation Systems","arxiv_id":"2311.09476","date":"2023-11-16","proceeding":null,"authors":["Jon Saad-Falcon","Omar Khattab","Christopher Potts","Matei Zaharia"],"abstract":"Evaluating retrieval-augmented generation (RAG) systems traditionally relies on hand annotations for input queries, passages to retrieve, and responses to generate. We introduce ARES, an Automated RAG Evaluation System, for evaluating RAG systems along the dimensions of context relevance, answer faithfulness, and answer relevance. By creating its own synthetic training data, ARES finetunes lightweight LM judges to assess the quality of individual RAG components. To mitigate potential prediction errors, ARES utilizes a small set of human-annotated datapoints for prediction-powered inference (PPI). Across eight different knowledge-intensive tasks in KILT, SuperGLUE, and AIS, ARES accurately evaluates RAG systems while using only a few hundred human annotations during evaluation. Furthermore, ARES judges remain effective across domain shifts, proving accurate even after changing the type of queries and/or documents used in the evaluated RAG systems. We make our code and datasets publicly available on Github.","url_abs":"https://arxiv.org/abs/2311.09476v2","url_pdf":"https://arxiv.org/pdf/2311.09476v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"ares-an-automated-evaluation-framework-for","repo_url":"https://github.com/stanford-futuredata/ares","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"rag","task_name":"RAG"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"retrieval-augmented-generation","task_name":"Retrieval-augmented Generation"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bart","method_name":"BART"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"rag","method_name":"RAG"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"set","method_name":"SET"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2311.09476","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.09476"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/stanford-futuredata/ares","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":13,"unverified":2},"by_repo_kind":{"official":{"samples":15,"ran":13,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"be55464f5573337e","entry":"clean_document","repo":"stanford-futuredata/ares","repo_kind":"official","path":"ares/LLM_as_a_Judge_Adaptation/Generate_Synthetic_Queries_and_Answers.py","file_url":"https://github.com/stanford-futuredata/ares/blob/HEAD/ares/LLM_as_a_Judge_Adaptation/Generate_Synthetic_Queries_and_Answers.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"be55464f5573337e"}},{"code_sha256_prefix":"7c82adfbee243a31","entry":"combine_query_document","repo":"stanford-futuredata/ares","repo_kind":"official","path":"ares/LLM_as_a_Judge_Adaptation/General_Binary_Classifier.py","file_url":"https://github.com/stanford-futuredata/ares/blob/HEAD/ares/LLM_as_a_Judge_Adaptation/General_Binary_Classifier.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7c82adfbee243a31"}},{"code_sha256_prefix":"61ab13d53232f752","entry":"extract_query","repo":"stanford-futuredata/ares","repo_kind":"official","path":"ares/ues_idp.py","file_url":"https://github.com/stanford-futuredata/ares/blob/HEAD/ares/ues_idp.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"61ab13d53232f752"}},{"code_sha256_prefix":"73c0cc4134a54844","entry":"format_text_for_fine_tuning_content_relevance_sequence_classification","repo":"stanford-futuredata/ares","repo_kind":"official","path":"ares/LLM_as_a_Judge_Adaptation/General_Binary_Classifier.py","file_url":"https://github.com/stanford-futuredata/ares/blob/HEAD/ares/LLM_as_a_Judge_Adaptation/General_Binary_Classifier.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"73c0cc4134a54844"}},{"code_sha256_prefix":"54900b5472f388a3","entry":"generate_answer_llm_approach","repo":"stanford-futuredata/ares","repo_kind":"official","path":"ares/LLM_as_a_Judge_Adaptation/LLM_Generation_Functions.py","file_url":"https://github.com/stanford-futuredata/ares/blob/HEAD/ares/LLM_as_a_Judge_Adaptation/LLM_Generation_Functions.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"54900b5472f388a3"}},{"code_sha256_prefix":"81baf8a93919f0bf","entry":"generate_synthetic_query_azure_approach","repo":"stanford-futuredata/ares","repo_kind":"official","path":"ares/LLM_as_a_Judge_Adaptation/LLM_Synthetic_Generation.py","file_url":"https://github.com/stanford-futuredata/ares/blob/HEAD/ares/LLM_as_a_Judge_Adaptation/LLM_Synthetic_Generation.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"81baf8a93919f0bf"}},{"code_sha256_prefix":"3c6369d4f5e9351e","entry":"generate_synthetic_query_llm_approach","repo":"stanford-futuredata/ares","repo_kind":"official","path":"ares/LLM_as_a_Judge_Adaptation/LLM_Generation_Functions.py","file_url":"https://github.com/stanford-futuredata/ares/blob/HEAD/ares/LLM_as_a_Judge_Adaptation/LLM_Generation_Functions.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3c6369d4f5e9351e"}},{"code_sha256_prefix":"56ba849663090e79","entry":"generate_synthetic_query_openai_approach","repo":"stanford-futuredata/ares","repo_kind":"official","path":"ares/LLM_as_a_Judge_Adaptation/LLM_Generation_Functions.py","file_url":"https://github.com/stanford-futuredata/ares/blob/HEAD/ares/LLM_as_a_Judge_Adaptation/LLM_Generation_Functions.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"56ba849663090e79"}},{"code_sha256_prefix":"67d0f16bdaed8f97","entry":"generate_synthetic_query_vllm_approach","repo":"stanford-futuredata/ares","repo_kind":"official","path":"ares/LLM_as_a_Judge_Adaptation/LLM_Synthetic_Generation.py","file_url":"https://github.com/stanford-futuredata/ares/blob/HEAD/ares/LLM_as_a_Judge_Adaptation/LLM_Synthetic_Generation.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"67d0f16bdaed8f97"}},{"code_sha256_prefix":"0b7eddc443d1a412","entry":"load_model","repo":"stanford-futuredata/ares","repo_kind":"official","path":"ares/LLM_as_a_Judge_Adaptation/Generate_Synthetic_Queries_and_Answers.py","file_url":"https://github.com/stanford-futuredata/ares/blob/HEAD/ares/LLM_as_a_Judge_Adaptation/Generate_Synthetic_Queries_and_Answers.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0b7eddc443d1a412"}},{"code_sha256_prefix":"042e539c6d8bd3de","entry":"tokenize_function","repo":"stanford-futuredata/ares","repo_kind":"official","path":"ares/LLM_as_a_Judge_Adaptation/General_Binary_Classifier.py","file_url":"https://github.com/stanford-futuredata/ares/blob/HEAD/ares/LLM_as_a_Judge_Adaptation/General_Binary_Classifier.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"042e539c6d8bd3de"}},{"code_sha256_prefix":"76a2c2ec78a2884c","entry":"validate_input_file","repo":"stanford-futuredata/ares","repo_kind":"official","path":"ares/LLM_as_a_Judge_Adaptation/Generate_Synthetic_Queries_and_Answers.py","file_url":"https://github.com/stanford-futuredata/ares/blob/HEAD/ares/LLM_as_a_Judge_Adaptation/Generate_Synthetic_Queries_and_Answers.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"76a2c2ec78a2884c"}},{"code_sha256_prefix":"9174b9f97d72aed9","entry":"validate_inputs","repo":"stanford-futuredata/ares","repo_kind":"official","path":"ares/ues_idp.py","file_url":"https://github.com/stanford-futuredata/ares/blob/HEAD/ares/ues_idp.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"9174b9f97d72aed9"}},{"code_sha256_prefix":"591b6449ad73d522","entry":"filter_synthetic_queries","repo":"stanford-futuredata/ares","repo_kind":"official","path":"ares/LLM_as_a_Judge_Adaptation/Filter_Synthetic_Queries.py","file_url":"https://github.com/stanford-futuredata/ares/blob/HEAD/ares/LLM_as_a_Judge_Adaptation/Filter_Synthetic_Queries.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"591b6449ad73d522"}},{"code_sha256_prefix":"30599a80b9dc6909","entry":"get_embedding","repo":"stanford-futuredata/ares","repo_kind":"official","path":"ares/LLM_as_a_Judge_Adaptation/Filter_Synthetic_Queries.py","file_url":"https://github.com/stanford-futuredata/ares/blob/HEAD/ares/LLM_as_a_Judge_Adaptation/Filter_Synthetic_Queries.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"30599a80b9dc6909"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}