{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/unirag-universal-retrieval-augmentation-for","title":"UniRAG: Universal Retrieval Augmentation for Large Vision Language Models","arxiv_id":"2405.10311","date":"2024-05-16","proceeding":null,"authors":["Sahel Sharifymoghaddam","Shivani Upadhyay","Wenhu Chen","Jimmy Lin"],"abstract":"Recently, Large Vision Language Models (LVLMs) have unlocked many complex use cases that require Multi-Modal (MM) understanding (e.g., image captioning or visual question answering) and MM generation (e.g., text-guided image generation or editing) capabilities. To further improve the output fidelityof LVLMs we introduce UniRAG, a plug-and-play technique that adds relevant retrieved information to prompts as few-shot examples during inference. Unlike the common belief that Retrieval Augmentation (RA) mainly improves generation or understanding of uncommon entities, our evaluation results on the MSCOCO dataset with common entities show that both proprietary models like GPT-4o and Gemini-Pro and smaller open-source models like LLaVA, LaVIT, and Emu2 significantly enhance their generation quality when their input prompts are augmented with relevant information retrieved by Vision-Language (VL) retrievers like UniIR models. All the necessary code to reproduce our results is available at https://github.com/castorini/UniRAG","url_abs":"https://arxiv.org/abs/2405.10311v3","url_pdf":"https://arxiv.org/pdf/2405.10311v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"unirag-universal-retrieval-augmentation-for","repo_url":"https://github.com/castorini/unirag","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"image-captioning","task_name":"Image Captioning"},{"task_slug":"image-generation","task_name":"Image Generation"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2405.10311","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2405.10311"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/castorini/unirag","reach":{"status":"ok"}}],"summary":{"ran":12,"unverified":1},"by_repo_kind":{"official":{"samples":13,"ran":12,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":13,"samples":[{"code_sha256_prefix":"d53d25a47fbce248","entry":"convert_to_tokenizer_input_format","repo":"castorini/unirag","repo_kind":"official","path":"src/unirag/eval_caption_generation.py","file_url":"https://github.com/castorini/unirag/blob/HEAD/src/unirag/eval_caption_generation.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"d53d25a47fbce248"}},{"code_sha256_prefix":"1606cd36064dba35","entry":"file_in_dir","repo":"castorini/unirag","repo_kind":"official","path":"src/unirag/eval_caption_generation.py","file_url":"https://github.com/castorini/unirag/blob/HEAD/src/unirag/eval_caption_generation.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1606cd36064dba35"}},{"code_sha256_prefix":"142c53c9cf9f8446","entry":"filter_qids","repo":"castorini/unirag","repo_kind":"official","path":"src/unirag/incontext_example_provider.py","file_url":"https://github.com/castorini/unirag/blob/HEAD/src/unirag/incontext_example_provider.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"142c53c9cf9f8446"}},{"code_sha256_prefix":"df3c779782a41086","entry":"get_candidates","repo":"castorini/unirag","repo_kind":"official","path":"src/unirag/incontext_example_provider.py","file_url":"https://github.com/castorini/unirag/blob/HEAD/src/unirag/incontext_example_provider.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"df3c779782a41086"}},{"code_sha256_prefix":"29a3bd6adf269138","entry":"get_positive_cands","repo":"castorini/unirag","repo_kind":"official","path":"src/unirag/incontext_example_provider.py","file_url":"https://github.com/castorini/unirag/blob/HEAD/src/unirag/incontext_example_provider.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"29a3bd6adf269138"}},{"code_sha256_prefix":"aa1c0514a26caa69","entry":"infer_lavit","repo":"castorini/unirag","repo_kind":"official","path":"src/unirag/image_generation_inference.py","file_url":"https://github.com/castorini/unirag/blob/HEAD/src/unirag/image_generation_inference.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"aa1c0514a26caa69"}},{"code_sha256_prefix":"cf46d511fdda3462","entry":"load_candidates","repo":"castorini/unirag","repo_kind":"official","path":"src/unirag/prepare_dataset.py","file_url":"https://github.com/castorini/unirag/blob/HEAD/src/unirag/prepare_dataset.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"cf46d511fdda3462"}},{"code_sha256_prefix":"3277243808d631aa","entry":"load_images","repo":"castorini/unirag","repo_kind":"official","path":"src/unirag/eval_image_generation.py","file_url":"https://github.com/castorini/unirag/blob/HEAD/src/unirag/eval_image_generation.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3277243808d631aa"}},{"code_sha256_prefix":"1d3391d39efd8b4c","entry":"load_prompts","repo":"castorini/unirag","repo_kind":"official","path":"src/unirag/eval_image_generation.py","file_url":"https://github.com/castorini/unirag/blob/HEAD/src/unirag/eval_image_generation.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"1d3391d39efd8b4c"}},{"code_sha256_prefix":"6a7fd654c9b1931c","entry":"load_retrieved_images","repo":"castorini/unirag","repo_kind":"official","path":"src/unirag/eval_image_generation.py","file_url":"https://github.com/castorini/unirag/blob/HEAD/src/unirag/eval_image_generation.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6a7fd654c9b1931c"}},{"code_sha256_prefix":"98c063b347cbdd01","entry":"sample_queries","repo":"castorini/unirag","repo_kind":"official","path":"src/unirag/prepare_dataset.py","file_url":"https://github.com/castorini/unirag/blob/HEAD/src/unirag/prepare_dataset.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"98c063b347cbdd01"}},{"code_sha256_prefix":"58629c6a6cb6b282","entry":"save_result","repo":"castorini/unirag","repo_kind":"official","path":"src/unirag/utils.py","file_url":"https://github.com/castorini/unirag/blob/HEAD/src/unirag/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"58629c6a6cb6b282"}},{"code_sha256_prefix":"643539116e049080","entry":"download_cached_file","repo":"castorini/unirag","repo_kind":"official","path":"src/unirag/utils.py","file_url":"https://github.com/castorini/unirag/blob/HEAD/src/unirag/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"643539116e049080"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}