{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/towards-text-image-interleaved-retrieval","title":"Towards Text-Image Interleaved Retrieval","arxiv_id":"2502.12799","date":"2025-02-18","proceeding":null,"authors":["Xin Zhang","Ziqi Dai","Yongqi Li","Yanzhao Zhang","Dingkun Long","Pengjun Xie","Meishan Zhang","Jun Yu","Wenjie Li","Min Zhang"],"abstract":"Current multimodal information retrieval studies mainly focus on single-image inputs, which limits real-world applications involving multiple images and text-image interleaved content. In this work, we introduce the text-image interleaved retrieval (TIIR) task, where the query and document are interleaved text-image sequences, and the model is required to understand the semantics from the interleaved context for effective retrieval. We construct a TIIR benchmark based on naturally interleaved wikiHow tutorials, where a specific pipeline is designed to generate interleaved queries. To explore the task, we adapt several off-the-shelf retrievers and build a dense baseline by interleaved multimodal large language model (MLLM). We then propose a novel Matryoshka Multimodal Embedder (MME), which compresses the number of visual tokens at different granularity, to address the challenge of excessive visual tokens in MLLM-based TIIR models. Experiments demonstrate that simple adaption of existing models does not consistently yield effective results. Our MME achieves significant improvements over the baseline by substantially fewer visual tokens. We provide extensive analysis and will release the dataset and code to facilitate future research.","url_abs":"https://arxiv.org/abs/2502.12799v1","url_pdf":"https://arxiv.org/pdf/2502.12799v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"towards-text-image-interleaved-retrieval","repo_url":"https://github.com/vec-ai/wikihow-tiir","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"none","reach":null}],"tasks":[{"task_slug":"information-retrieval","task_name":"Information Retrieval"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"large-language-model","task_name":"Large Language Model"},{"task_slug":"mme","task_name":"MME"},{"task_slug":"multimodal-large-language-model","task_name":"Multimodal Large Language Model"},{"task_slug":"retrieval","task_name":"Retrieval"}],"methods":[{"method_slug":"focus","method_name":"Focus"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2502.12799","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2502.12799"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/HumanSignal/label-studio","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":2,"unverified":4},"by_repo_kind":{"found_in_text":{"samples":6,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"0ddad62c4329b3ef","entry":"get_fields","repo":"HumanSignal/label-studio","repo_kind":"found_in_text","path":"label_studio/core/bulk_update_utils.py","file_url":"https://github.com/HumanSignal/label-studio/blob/HEAD/label_studio/core/bulk_update_utils.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0ddad62c4329b3ef"}},{"code_sha256_prefix":"f942b532e3a9a0fa","entry":"override_report_only_csp","repo":"HumanSignal/label-studio","repo_kind":"found_in_text","path":"label_studio/core/decorators.py","file_url":"https://github.com/HumanSignal/label-studio/blob/HEAD/label_studio/core/decorators.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f942b532e3a9a0fa"}},{"code_sha256_prefix":"03d1e40272750703","entry":"authorization_header_from_x_api_key","repo":"HumanSignal/label-studio","repo_kind":"found_in_text","path":"label_studio/core/middleware.py","file_url":"https://github.com/HumanSignal/label-studio/blob/HEAD/label_studio/core/middleware.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"03d1e40272750703"}},{"code_sha256_prefix":"e74c059f314bd944","entry":"flatten","repo":"HumanSignal/label-studio","repo_kind":"found_in_text","path":"label_studio/core/bulk_update_utils.py","file_url":"https://github.com/HumanSignal/label-studio/blob/HEAD/label_studio/core/bulk_update_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e74c059f314bd944"}},{"code_sha256_prefix":"fd1c36b7237ed788","entry":"parse_config_to_xml","repo":"HumanSignal/label-studio","repo_kind":"found_in_text","path":"label_studio/core/label_config.py","file_url":"https://github.com/HumanSignal/label-studio/blob/HEAD/label_studio/core/label_config.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fd1c36b7237ed788"}},{"code_sha256_prefix":"3456dfe51a2a195e","entry":"parse_input_args","repo":"HumanSignal/label-studio","repo_kind":"found_in_text","path":"label_studio/core/argparser.py","file_url":"https://github.com/HumanSignal/label-studio/blob/HEAD/label_studio/core/argparser.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3456dfe51a2a195e"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}