{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/wecromcl-weakly-supervised-cross-modality","title":"WeCromCL: Weakly Supervised Cross-Modality Contrastive Learning for Transcription-only Supervised Text Spotting","arxiv_id":"2407.19507","date":"2024-07-28","proceeding":null,"authors":["Jingjing Wu","Zhengyao Fang","Pengyuan Lyu","Chengquan Zhang","Fanglin Chen","Guangming Lu","Wenjie Pei"],"abstract":"Transcription-only Supervised Text Spotting aims to learn text spotters relying only on transcriptions but no text boundaries for supervision, thus eliminating expensive boundary annotation. The crux of this task lies in locating each transcription in scene text images without location annotations. In this work, we formulate this challenging problem as a Weakly Supervised Cross-modality Contrastive Learning problem, and design a simple yet effective model dubbed WeCromCL that is able to detect each transcription in a scene image in a weakly supervised manner. Unlike typical methods for cross-modality contrastive learning that focus on modeling the holistic semantic correlation between an entire image and a text description, our WeCromCL conducts atomistic contrastive learning to model the character-wise appearance consistency between a text transcription and its correlated region in a scene image to detect an anchor point for the transcription in a weakly supervised manner. The detected anchor points by WeCromCL are further used as pseudo location labels to guide the learning of text spotting. Extensive experiments on four challenging benchmarks demonstrate the superior performance of our model over other methods. Code will be released.","url_abs":"https://arxiv.org/abs/2407.19507v2","url_pdf":"https://arxiv.org/pdf/2407.19507v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"wecromcl-weakly-supervised-cross-modality","repo_url":"https://github.com/ZhengyaoFang/WeCromCL","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"contrastive-learning","task_name":"Contrastive Learning"},{"task_slug":"text-spotting","task_name":"Text Spotting"}],"methods":[{"method_slug":"contrastive-learning","method_name":"Contrastive Learning"},{"method_slug":"focus","method_name":"Focus"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2407.19507","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2407.19507"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ZhengyaoFang/WeCromCL","reach":null}],"summary":{"ran":5,"ran_draft_wrong":3,"unverified":3},"by_repo_kind":{"official":{"samples":10,"ran":7,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":11,"samples":[{"code_sha256_prefix":"e5366d67a673e8a0","entry":"AttentionPool2d_text_as_query","repo":"ZhengyaoFang/WeCromCL","repo_kind":"official","path":"models/wecromcl.py","file_url":"https://github.com/ZhengyaoFang/WeCromCL/blob/HEAD/models/wecromcl.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e5366d67a673e8a0"}},{"code_sha256_prefix":"bbe572b0398d293e","entry":"BiFPN","repo":"ZhengyaoFang/WeCromCL","repo_kind":"official","path":"models/wecromcl.py","file_url":"https://github.com/ZhengyaoFang/WeCromCL/blob/HEAD/models/wecromcl.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"bbe572b0398d293e"}},{"code_sha256_prefix":"992ac4a7af311b77","entry":"DeformableTransformerEncoder","repo":"ZhengyaoFang/WeCromCL","repo_kind":"official","path":"models/wecromcl.py","file_url":"https://github.com/ZhengyaoFang/WeCromCL/blob/HEAD/models/wecromcl.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"992ac4a7af311b77"}},{"code_sha256_prefix":"05b9a47ccafbfbbb","entry":"MSDeformAttn","repo":"ZhengyaoFang/WeCromCL","repo_kind":"official","path":"models/wecromcl.py","file_url":"https://github.com/ZhengyaoFang/WeCromCL/blob/HEAD/models/wecromcl.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"05b9a47ccafbfbbb"}},{"code_sha256_prefix":"54fc7a4215b66788","entry":"PositionalEncoding2D","repo":"ZhengyaoFang/WeCromCL","repo_kind":"official","path":"models/wecromcl.py","file_url":"https://github.com/ZhengyaoFang/WeCromCL/blob/HEAD/models/wecromcl.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"54fc7a4215b66788"}},{"code_sha256_prefix":"3b56bcda10bf41fb","entry":"_get_clones","repo":"ZhengyaoFang/WeCromCL","repo_kind":"official","path":"models/wecromcl.py","file_url":"https://github.com/ZhengyaoFang/WeCromCL/blob/HEAD/models/wecromcl.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3b56bcda10bf41fb"}},{"code_sha256_prefix":"dbcb53696bc43ef9","entry":"conv3x3","repo":null,"repo_kind":null,"path":null,"file_url":null,"link_basis":"identical_code_first_harvested_elsewhere","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":null,"inline_ok":false,"mcp_get_code":{"code_sha256":"dbcb53696bc43ef9"}},{"code_sha256_prefix":"fca7e85d958618dd","entry":"conv3x3_bn_relu","repo":"ZhengyaoFang/WeCromCL","repo_kind":"official","path":"models/wecromcl.py","file_url":"https://github.com/ZhengyaoFang/WeCromCL/blob/HEAD/models/wecromcl.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"fca7e85d958618dd"}},{"code_sha256_prefix":"41dddcf79472d123","entry":"DeformableTransformerEncoderLayer","repo":"ZhengyaoFang/WeCromCL","repo_kind":"official","path":"models/wecromcl.py","file_url":"https://github.com/ZhengyaoFang/WeCromCL/blob/HEAD/models/wecromcl.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"41dddcf79472d123"}},{"code_sha256_prefix":"f5040579df9ccf04","entry":"Feature_encoder","repo":"ZhengyaoFang/WeCromCL","repo_kind":"official","path":"models/wecromcl.py","file_url":"https://github.com/ZhengyaoFang/WeCromCL/blob/HEAD/models/wecromcl.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"f5040579df9ccf04"}},{"code_sha256_prefix":"6cafbbf3a6c3a81c","entry":"WeCromCL","repo":"ZhengyaoFang/WeCromCL","repo_kind":"official","path":"models/wecromcl.py","file_url":"https://github.com/ZhengyaoFang/WeCromCL/blob/HEAD/models/wecromcl.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"6cafbbf3a6c3a81c"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}