{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/self-supervised-vision-language-pretraining","title":"Self-supervised vision-language pretraining for Medical visual question answering","arxiv_id":"2211.13594","date":"2022-11-24","proceeding":null,"authors":["Pengfei Li","Gang Liu","Lin Tan","Jinying Liao","Shenjun Zhong"],"abstract":"Medical image visual question answering (VQA) is a task to answer clinical questions, given a radiographic image, which is a challenging problem that requires a model to integrate both vision and language information. To solve medical VQA problems with a limited number of training data, pretrain-finetune paradigm is widely used to improve the model generalization. In this paper, we propose a self-supervised method that applies Masked image modeling, Masked language modeling, Image text matching and Image text alignment via contrastive learning (M2I2) for pretraining on medical image caption dataset, and finetunes to downstream medical VQA tasks. The proposed method achieves state-of-the-art performance on all the three public medical VQA datasets. Our codes and models are available at https://github.com/pengfeiliHEU/M2I2.","url_abs":"https://arxiv.org/abs/2211.13594v1","url_pdf":"https://arxiv.org/pdf/2211.13594v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"self-supervised-vision-language-pretraining","repo_url":"https://github.com/pengfeiliheu/m2i2","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"self-supervised-vision-language-pretraining","repo_url":"https://github.com/pengfeiliheu/mumc","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"contrastive-learning","task_name":"Contrastive Learning"},{"task_slug":"image-text-matching","task_name":"Image-text matching"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"masked-language-modeling","task_name":"Masked Language Modeling"},{"task_slug":"medical-visual-question-answering","task_name":"Medical Visual Question Answering"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"text-matching","task_name":"Text Matching"},{"task_slug":"visual-question-answering-1","task_name":"Visual Question Answering"},{"task_slug":"visual-question-answering","task_name":"Visual Question Answering (VQA)"}],"methods":[{"method_slug":"contrastive-learning","method_name":"Contrastive Learning"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2211.13594","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2211.13594"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/pengfeiliheu/m2i2","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/pengfeiliheu/mumc","reach":{"status":"ok"}}],"summary":{"ran_draft_wrong":4,"ran_fixture":1,"ran":1,"unverified":4},"by_repo_kind":{"official":{"samples":10,"ran":6,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"cf9ffa02a42184af","entry":"whitespace_tokenize","repo":"pengfeiliheu/m2i2","repo_kind":"official","path":"models/tokenization_bert.py","file_url":"https://github.com/pengfeiliheu/m2i2/blob/HEAD/models/tokenization_bert.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":2,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"cf9ffa02a42184af"}},{"code_sha256_prefix":"aa81d03f07735ab8","entry":"compute_acc","repo":"pengfeiliheu/m2i2","repo_kind":"official","path":"utils.py","file_url":"https://github.com/pengfeiliheu/m2i2/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"aa81d03f07735ab8"}},{"code_sha256_prefix":"ad12a494674d23fb","entry":"compute_n_params","repo":"pengfeiliheu/m2i2","repo_kind":"official","path":"utils.py","file_url":"https://github.com/pengfeiliheu/m2i2/blob/HEAD/utils.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"ad12a494674d23fb"}},{"code_sha256_prefix":"c6ec173f19f5c34d","entry":"interpolate_pos_embed","repo":"pengfeiliheu/m2i2","repo_kind":"official","path":"models/mae.py","file_url":"https://github.com/pengfeiliheu/m2i2/blob/HEAD/models/mae.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c6ec173f19f5c34d"}},{"code_sha256_prefix":"e7fbc7a74a3457c7","entry":"load_vocab","repo":"pengfeiliheu/m2i2","repo_kind":"official","path":"models/tokenization_bert.py","file_url":"https://github.com/pengfeiliheu/m2i2/blob/HEAD/models/tokenization_bert.py","link_basis":"harvester_set","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e7fbc7a74a3457c7"}},{"code_sha256_prefix":"aa5a91226269bf75","entry":"tile","repo":"pengfeiliheu/m2i2","repo_kind":"official","path":"models/model_vqa.py","file_url":"https://github.com/pengfeiliheu/m2i2/blob/HEAD/models/model_vqa.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"aa5a91226269bf75"}},{"code_sha256_prefix":"73cecca9f3575f09","entry":"concat_all_gather","repo":"pengfeiliheu/m2i2","repo_kind":"official","path":"models/model_pretrain.py","file_url":"https://github.com/pengfeiliheu/m2i2/blob/HEAD/models/model_pretrain.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"73cecca9f3575f09"}},{"code_sha256_prefix":"3185afc3e87293ed","entry":"get_2d_sincos_pos_embed","repo":"pengfeiliheu/m2i2","repo_kind":"official","path":"models/mae.py","file_url":"https://github.com/pengfeiliheu/m2i2/blob/HEAD/models/mae.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3185afc3e87293ed"}},{"code_sha256_prefix":"f10004e059714d42","entry":"get_2d_sincos_pos_embed_from_grid","repo":"pengfeiliheu/m2i2","repo_kind":"official","path":"models/mae.py","file_url":"https://github.com/pengfeiliheu/m2i2/blob/HEAD/models/mae.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f10004e059714d42"}},{"code_sha256_prefix":"26be70dca3249c0b","entry":"load_tf_weights_in_bert","repo":"pengfeiliheu/m2i2","repo_kind":"official","path":"models/xbert.py","file_url":"https://github.com/pengfeiliheu/m2i2/blob/HEAD/models/xbert.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"26be70dca3249c0b"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}