{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/joint-learning-of-localized-representations","title":"Joint Learning of Localized Representations from Medical Images and Reports","arxiv_id":"2112.02889","date":"2021-12-06","proceeding":null,"authors":["Philip Müller","Georgios Kaissis","Congyu Zou","Daniel Rueckert"],"abstract":"Contrastive learning has proven effective for pre-training image models on unlabeled data with promising results for tasks such as medical image classification. Using paired text (like radiological reports) during pre-training improves the results even further. Still, most existing methods target image classification downstream tasks and may not be optimal for localized tasks like semantic segmentation or object detection. We therefore propose Localized representation learning from Vision and Text (LoVT), to our best knowledge, the first text-supervised pre-training method that targets localized medical imaging tasks. Our method combines instance-level image-report contrastive learning with local contrastive learning on image region and report sentence representations. We evaluate LoVT and commonly used pre-training methods on an evaluation framework of 18 localized tasks on chest X-rays from five public datasets. LoVT performs best on 10 of the 18 studied tasks making it the preferred method of choice for localized tasks.","url_abs":"https://arxiv.org/abs/2112.02889v2","url_pdf":"https://arxiv.org/pdf/2112.02889v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"joint-learning-of-localized-representations","repo_url":"https://github.com/philip-mueller/lovt","is_official":1,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"contrastive-learning","task_name":"Contrastive Learning"},{"task_slug":"medical-image-classification","task_name":"Medical Image Classification"},{"task_slug":"object-detection","task_name":"Object Detection"},{"task_slug":"representation-learning","task_name":"Representation Learning"},{"task_slug":"semantic-segmentation","task_name":"Semantic Segmentation"},{"task_slug":"sentence","task_name":"Sentence"},{"task_slug":"image-classification","task_name":"image-classification"},{"task_slug":"object-detection-1","task_name":"object-detection"}],"methods":[{"method_slug":"contrastive-learning","method_name":"Contrastive Learning"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2112.02889","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2112.02889"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/philip-mueller/lovt","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":7},"by_repo_kind":{"official":{"samples":7,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"3f4eeb7293e3ee40","entry":"get_paper_data","repo":"philip-mueller/lovt","repo_kind":"official","path":"src/analysis/data_exporter.py","file_url":"https://github.com/philip-mueller/lovt/blob/HEAD/src/analysis/data_exporter.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3f4eeb7293e3ee40"}},{"code_sha256_prefix":"9e6308c42a9264cb","entry":"get_run_id_from_path","repo":"philip-mueller/lovt","repo_kind":"official","path":"src/analysis/postprocess_run.py","file_url":"https://github.com/philip-mueller/lovt/blob/HEAD/src/analysis/postprocess_run.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9e6308c42a9264cb"}},{"code_sha256_prefix":"43fe0139d6077ac9","entry":"init_loggers","repo":"philip-mueller/lovt","repo_kind":"official","path":"src/common/script_utils.py","file_url":"https://github.com/philip-mueller/lovt/blob/HEAD/src/common/script_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"43fe0139d6077ac9"}},{"code_sha256_prefix":"e342c1ca604a089e","entry":"init_trainer","repo":"philip-mueller/lovt","repo_kind":"official","path":"src/common/script_utils.py","file_url":"https://github.com/philip-mueller/lovt/blob/HEAD/src/common/script_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e342c1ca604a089e"}},{"code_sha256_prefix":"0871700b35e8aa93","entry":"load_backbone","repo":"philip-mueller/lovt","repo_kind":"official","path":"src/baselines/baseline_utils.py","file_url":"https://github.com/philip-mueller/lovt/blob/HEAD/src/baselines/baseline_utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0871700b35e8aa93"}},{"code_sha256_prefix":"d18d37af2901af10","entry":"load_downstream_results","repo":"philip-mueller/lovt","repo_kind":"official","path":"src/analysis/data_exporter.py","file_url":"https://github.com/philip-mueller/lovt/blob/HEAD/src/analysis/data_exporter.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d18d37af2901af10"}},{"code_sha256_prefix":"d693847dd3d12d01","entry":"load_results","repo":"philip-mueller/lovt","repo_kind":"official","path":"src/analysis/data_exporter.py","file_url":"https://github.com/philip-mueller/lovt/blob/HEAD/src/analysis/data_exporter.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d693847dd3d12d01"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}