{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/x-instructblip-a-framework-for-aligning-x","title":"X-InstructBLIP: A Framework for aligning X-Modal instruction-aware representations to LLMs and Emergent Cross-modal Reasoning","arxiv_id":"2311.18799","date":"2023-11-30","proceeding":null,"authors":["Artemis Panagopoulou","Le Xue","Ning Yu","Junnan Li","Dongxu Li","Shafiq Joty","ran Xu","Silvio Savarese","Caiming Xiong","Juan Carlos Niebles"],"abstract":"Recent research has achieved significant advancements in visual reasoning tasks through learning image-to-language projections and leveraging the impressive reasoning abilities of Large Language Models (LLMs). This paper introduces an efficient and effective framework that integrates multiple modalities (images, 3D, audio and video) to a frozen LLM and demonstrates an emergent ability for cross-modal reasoning (2+ modality inputs). Our approach explores two distinct projection mechanisms: Q-Formers and Linear Projections (LPs). Through extensive experimentation across all four modalities on 16 benchmarks, we explore both methods and assess their adaptability in integrated and separate cross-modal reasoning. The Q-Former projection demonstrates superior performance in single modality scenarios and adaptability in joint versus discriminative reasoning involving two or more modalities. However, it exhibits lower generalization capabilities than linear projection in contexts where task-modality data are limited. To enable this framework, we devise a scalable pipeline that automatically generates high-quality, instruction-tuning datasets from readily available captioning data across different modalities, and contribute 24K QA data for audio and 250K QA data for 3D. To facilitate further research in cross-modal reasoning, we introduce the DisCRn (Discriminative Cross-modal Reasoning) benchmark comprising 9K audio-video QA samples and 28K image-3D QA samples that require the model to reason discriminatively across disparate input modalities.","url_abs":"https://arxiv.org/abs/2311.18799v2","url_pdf":"https://arxiv.org/pdf/2311.18799v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"x-instructblip-a-framework-for-aligning-x","repo_url":"https://github.com/artemisp/lavis-xinstructblip","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"BSD-3-Clause"}},{"paper_slug":"x-instructblip-a-framework-for-aligning-x","repo_url":"https://github.com/salesforce/lavis","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"BSD-3-Clause"}}],"tasks":[{"task_slug":"visual-reasoning","task_name":"Visual Reasoning"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2311.18799","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2311.18799"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/salesforce/lavis","reach":{"status":"ok","spdx":"BSD-3-Clause"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/artemisp/lavis-xinstructblip","reach":{"status":"ok","spdx":"BSD-3-Clause"}}],"summary":{"ran_fixture":1,"ran_draft_wrong":1,"ran":3,"unverified":2},"by_repo_kind":{"official":{"samples":7,"ran":5,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"7d214655954e2bc6","entry":"getAttMap","repo":"artemisp/lavis-xinstructblip","repo_kind":"official","path":"lavis/common/gradcam.py","file_url":"https://github.com/artemisp/lavis-xinstructblip/blob/HEAD/lavis/common/gradcam.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"7d214655954e2bc6"}},{"code_sha256_prefix":"c6ec173f19f5c34d","entry":"interpolate_pos_embed","repo":"artemisp/lavis-xinstructblip","repo_kind":"official","path":"lavis/models/vit.py","file_url":"https://github.com/artemisp/lavis-xinstructblip/blob/HEAD/lavis/models/vit.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"c6ec173f19f5c34d"}},{"code_sha256_prefix":"98589643273920ba","entry":"main_process","repo":"artemisp/lavis-xinstructblip","repo_kind":"official","path":"lavis/common/dist_utils.py","file_url":"https://github.com/artemisp/lavis-xinstructblip/blob/HEAD/lavis/common/dist_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"98589643273920ba"}},{"code_sha256_prefix":"791c72070c0b1cfb","entry":"node_to_dict","repo":"artemisp/lavis-xinstructblip","repo_kind":"official","path":"lavis/common/config.py","file_url":"https://github.com/artemisp/lavis-xinstructblip/blob/HEAD/lavis/common/config.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"791c72070c0b1cfb"}},{"code_sha256_prefix":"cb33571427334815","entry":"tile","repo":"artemisp/lavis-xinstructblip","repo_kind":"official","path":"lavis/models/base_model.py","file_url":"https://github.com/artemisp/lavis-xinstructblip/blob/HEAD/lavis/models/base_model.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"cb33571427334815"}},{"code_sha256_prefix":"0ec9fc2025c16f65","entry":"all_gather_with_grad","repo":"artemisp/lavis-xinstructblip","repo_kind":"official","path":"lavis/models/base_model.py","file_url":"https://github.com/artemisp/lavis-xinstructblip/blob/HEAD/lavis/models/base_model.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"0ec9fc2025c16f65"}},{"code_sha256_prefix":"643539116e049080","entry":"download_cached_file","repo":"artemisp/lavis-xinstructblip","repo_kind":"official","path":"lavis/common/dist_utils.py","file_url":"https://github.com/artemisp/lavis-xinstructblip/blob/HEAD/lavis/common/dist_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"BSD-3-Clause","inline_ok":true,"mcp_get_code":{"code_sha256":"643539116e049080"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}