{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/quilt-llava-visual-instruction-tuning-by","title":"Quilt-LLaVA: Visual Instruction Tuning by Extracting Localized Narratives from Open-Source Histopathology Videos","arxiv_id":"2312.04746","date":"2023-12-07","proceeding":"CVPR 2024 1","authors":["Mehmet Saygin Seyfioglu","Wisdom O. Ikezogwo","Fatemeh Ghezloo","Ranjay Krishna","Linda Shapiro"],"abstract":"Diagnosis in histopathology requires a global whole slide images (WSIs) analysis, requiring pathologists to compound evidence from different WSI patches. The gigapixel scale of WSIs poses a challenge for histopathology multi-modal models. Training multi-model models for histopathology requires instruction tuning datasets, which currently contain information for individual image patches, without a spatial grounding of the concepts within each patch and without a wider view of the WSI. Therefore, they lack sufficient diagnostic capacity for histopathology. To bridge this gap, we introduce Quilt-Instruct, a large-scale dataset of 107,131 histopathology-specific instruction question/answer pairs, grounded within diagnostically relevant image patches that make up the WSI. Our dataset is collected by leveraging educational histopathology videos from YouTube, which provides spatial localization of narrations by automatically extracting the narrators' cursor positions. Quilt-Instruct supports contextual reasoning by extracting diagnosis and supporting facts from the entire WSI. Using Quilt-Instruct, we train Quilt-LLaVA, which can reason beyond the given single image patch, enabling diagnostic reasoning across patches. To evaluate Quilt-LLaVA, we propose a comprehensive evaluation dataset created from 985 images and 1283 human-generated question-answers. We also thoroughly evaluate Quilt-LLaVA using public histopathology datasets, where Quilt-LLaVA significantly outperforms SOTA by over 10% on relative GPT-4 score and 4% and 9% on open and closed set VQA. Our code, data, and model are publicly accessible at quilt-llava.github.io.","url_abs":"https://arxiv.org/abs/2312.04746v3","url_pdf":"https://arxiv.org/pdf/2312.04746v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"quilt-llava-visual-instruction-tuning-by","repo_url":"https://github.com/aldraus/quilt-llava","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"quilt-llava-visual-instruction-tuning-by","repo_url":"https://github.com/wisdomikezogwo/quilt1m","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"diagnostic","task_name":"Diagnostic"},{"task_slug":"image-captioning","task_name":"Image Captioning"},{"task_slug":"visual-question-answering","task_name":"Visual Question Answering (VQA)"},{"task_slug":"whole-slide-images","task_name":"whole slide images"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"focus","method_name":"Focus"},{"method_slug":"gpt-4","method_name":"GPT-4"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"position-wise-feed-forward-layer","method_name":"Position-Wise Feed-Forward Layer"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"set","method_name":"SET"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2312.04746","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.04746"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/wisdomikezogwo/quilt1m","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/aldraus/quilt-llava","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":3,"ran":1,"ran_violates":1,"unverified":7},"by_repo_kind":{"listed":{"samples":12,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"592b3c1a88f93d7c","entry":"expand2square","repo":"aldraus/quilt-llava","repo_kind":"listed","path":"llava/mm_utils.py","file_url":"https://github.com/aldraus/quilt-llava/blob/HEAD/llava/mm_utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"592b3c1a88f93d7c"}},{"code_sha256_prefix":"42a46570620cd9fa","entry":"get_chunk","repo":"aldraus/quilt-llava","repo_kind":"listed","path":"llava/eval/model_vqa.py","file_url":"https://github.com/aldraus/quilt-llava/blob/HEAD/llava/eval/model_vqa.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"42a46570620cd9fa"}},{"code_sha256_prefix":"c3ee9d07c900dd55","entry":"load_image_from_base64","repo":"aldraus/quilt-llava","repo_kind":"listed","path":"llava/mm_utils.py","file_url":"https://github.com/aldraus/quilt-llava/blob/HEAD/llava/mm_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c3ee9d07c900dd55"}},{"code_sha256_prefix":"763f6fe20bd52fbb","entry":"parse_score","repo":"aldraus/quilt-llava","repo_kind":"listed","path":"llava/eval/quilt_gpt_eval.py","file_url":"https://github.com/aldraus/quilt-llava/blob/HEAD/llava/eval/quilt_gpt_eval.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":1,"contract_check":"VIOLATES","metamorphic_tier":"well_formed","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"763f6fe20bd52fbb"}},{"code_sha256_prefix":"076c252c52cbb161","entry":"split_list","repo":"aldraus/quilt-llava","repo_kind":"listed","path":"llava/eval/model_vqa.py","file_url":"https://github.com/aldraus/quilt-llava/blob/HEAD/llava/eval/model_vqa.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"076c252c52cbb161"}},{"code_sha256_prefix":"43ef215b445193c1","entry":"brevity_penalty","repo":"aldraus/quilt-llava","repo_kind":"listed","path":"llava/eval/quilt_utils.py","file_url":"https://github.com/aldraus/quilt-llava/blob/HEAD/llava/eval/quilt_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"43ef215b445193c1"}},{"code_sha256_prefix":"0f12178b8ecbfaf0","entry":"evaluate","repo":"aldraus/quilt-llava","repo_kind":"listed","path":"llava/eval/quilt_eval.py","file_url":"https://github.com/aldraus/quilt-llava/blob/HEAD/llava/eval/quilt_eval.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0f12178b8ecbfaf0"}},{"code_sha256_prefix":"c00535a655c69753","entry":"get_eval","repo":"aldraus/quilt-llava","repo_kind":"listed","path":"llava/eval/quilt_gpt_eval.py","file_url":"https://github.com/aldraus/quilt-llava/blob/HEAD/llava/eval/quilt_gpt_eval.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c00535a655c69753"}},{"code_sha256_prefix":"77678f2758141df9","entry":"load_jsonl","repo":"aldraus/quilt-llava","repo_kind":"listed","path":"llava/eval/quilt_eval.py","file_url":"https://github.com/aldraus/quilt-llava/blob/HEAD/llava/eval/quilt_eval.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"77678f2758141df9"}},{"code_sha256_prefix":"21c8c6289af5c1e5","entry":"modified_precision","repo":"aldraus/quilt-llava","repo_kind":"listed","path":"llava/eval/quilt_utils.py","file_url":"https://github.com/aldraus/quilt-llava/blob/HEAD/llava/eval/quilt_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"21c8c6289af5c1e5"}},{"code_sha256_prefix":"344dff4791fd1381","entry":"process_images","repo":"aldraus/quilt-llava","repo_kind":"listed","path":"llava/mm_utils.py","file_url":"https://github.com/aldraus/quilt-llava/blob/HEAD/llava/mm_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"344dff4791fd1381"}},{"code_sha256_prefix":"2fd31a236724b42d","entry":"split_sentence","repo":"aldraus/quilt-llava","repo_kind":"listed","path":"llava/eval/quilt_utils.py","file_url":"https://github.com/aldraus/quilt-llava/blob/HEAD/llava/eval/quilt_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"2fd31a236724b42d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}