{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/mpmqa-multimodal-question-answering-on","title":"MPMQA: Multimodal Question Answering on Product Manuals","arxiv_id":"2304.09660","date":"2023-04-19","proceeding":null,"authors":["Liang Zhang","Anwen Hu","Jing Zhang","Shuo Hu","Qin Jin"],"abstract":"Visual contents, such as illustrations and images, play a big role in product manual understanding. Existing Product Manual Question Answering (PMQA) datasets tend to ignore visual contents and only retain textual parts. In this work, to emphasize the importance of multimodal contents, we propose a Multimodal Product Manual Question Answering (MPMQA) task. For each question, MPMQA requires the model not only to process multimodal contents but also to provide multimodal answers. To support MPMQA, a large-scale dataset PM209 is constructed with human annotations, which contains 209 product manuals from 27 well-known consumer electronic brands. Human annotations include 6 types of semantic regions for manual contents and 22,021 pairs of question and answer. Especially, each answer consists of a textual sentence and related visual regions from manuals. Taking into account the length of product manuals and the fact that a question is always related to a small number of pages, MPMQA can be naturally split into two subtasks: retrieving most related pages and then generating multimodal answers. We further propose a unified model that can perform these two subtasks all together and achieve comparable performance with multiple task-specific models. The PM209 dataset is available at https://github.com/AIM3-RUC/MPMQA.","url_abs":"https://arxiv.org/abs/2304.09660v1","url_pdf":"https://arxiv.org/pdf/2304.09660v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"mpmqa-multimodal-question-answering-on","repo_url":"https://github.com/aim3-ruc/mpmqa","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"sentence","task_name":"Sentence"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2304.09660","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2304.09660"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/aim3-ruc/mpmqa","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/AIM3-RUC/MPMQA","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"unverified":11},"by_repo_kind":{"official":{"samples":11,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"fc138667a09eb09a","entry":"compute_visual_answer_metics","repo":"AIM3-RUC/MPMQA","repo_kind":"official","path":"evaluate.py","file_url":"https://github.com/AIM3-RUC/MPMQA/blob/HEAD/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"fc138667a09eb09a"}},{"code_sha256_prefix":"00ea78fdd8ca164d","entry":"divide_box_grid","repo":"AIM3-RUC/MPMQA","repo_kind":"official","path":"dataset/utils.py","file_url":"https://github.com/AIM3-RUC/MPMQA/blob/HEAD/dataset/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"00ea78fdd8ca164d"}},{"code_sha256_prefix":"bd1100fba72bdfc2","entry":"get_mqa_loader","repo":"AIM3-RUC/MPMQA","repo_kind":"official","path":"dataset/mqa_dataset.py","file_url":"https://github.com/AIM3-RUC/MPMQA/blob/HEAD/dataset/mqa_dataset.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bd1100fba72bdfc2"}},{"code_sha256_prefix":"49aa5f1fc19f0eb5","entry":"merge_recall","repo":"AIM3-RUC/MPMQA","repo_kind":"official","path":"utils.py","file_url":"https://github.com/AIM3-RUC/MPMQA/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"49aa5f1fc19f0eb5"}},{"code_sha256_prefix":"483f5441b7f2bd89","entry":"mqa_collate_fn","repo":"AIM3-RUC/MPMQA","repo_kind":"official","path":"dataset/mqa_dataset.py","file_url":"https://github.com/AIM3-RUC/MPMQA/blob/HEAD/dataset/mqa_dataset.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"483f5441b7f2bd89"}},{"code_sha256_prefix":"4bfbb5d6eea393ac","entry":"mqa_contrast_collate_fn","repo":"AIM3-RUC/MPMQA","repo_kind":"official","path":"dataset/mqa_page_contrast.py","file_url":"https://github.com/AIM3-RUC/MPMQA/blob/HEAD/dataset/mqa_page_contrast.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4bfbb5d6eea393ac"}},{"code_sha256_prefix":"891802712c9626ab","entry":"obj_to_cuda","repo":"AIM3-RUC/MPMQA","repo_kind":"official","path":"utils.py","file_url":"https://github.com/AIM3-RUC/MPMQA/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"891802712c9626ab"}},{"code_sha256_prefix":"be0b02d6f8029207","entry":"pad_2d_mask","repo":"AIM3-RUC/MPMQA","repo_kind":"official","path":"dataset/utils.py","file_url":"https://github.com/AIM3-RUC/MPMQA/blob/HEAD/dataset/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"be0b02d6f8029207"}},{"code_sha256_prefix":"a0bb3c41bf1540de","entry":"pad_features","repo":"AIM3-RUC/MPMQA","repo_kind":"official","path":"models/utils.py","file_url":"https://github.com/AIM3-RUC/MPMQA/blob/HEAD/models/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a0bb3c41bf1540de"}},{"code_sha256_prefix":"48ee7bb6c76a6eee","entry":"remove_punc","repo":"AIM3-RUC/MPMQA","repo_kind":"official","path":"evaluate.py","file_url":"https://github.com/AIM3-RUC/MPMQA/blob/HEAD/evaluate.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"48ee7bb6c76a6eee"}},{"code_sha256_prefix":"c78ff121ca63a1a1","entry":"retrieval_eval","repo":"AIM3-RUC/MPMQA","repo_kind":"official","path":"utils.py","file_url":"https://github.com/AIM3-RUC/MPMQA/blob/HEAD/utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c78ff121ca63a1a1"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}