{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/vokenization-improving-language-understanding","title":"Vokenization: Improving Language Understanding with Contextualized, Visual-Grounded Supervision","arxiv_id":"2010.06775","date":"2020-10-14","proceeding":"EMNLP 2020 11","authors":["Hao Tan","Mohit Bansal"],"abstract":"Humans learn language by listening, speaking, writing, reading, and also, via interaction with the multimodal real world. Existing language pre-training frameworks show the effectiveness of text-only self-supervision while we explore the idea of a visually-supervised language model in this paper. We find that the main reason hindering this exploration is the large divergence in magnitude and distributions between the visually-grounded language datasets and pure-language corpora. Therefore, we develop a technique named \"vokenization\" that extrapolates multimodal alignments to language-only data by contextually mapping language tokens to their related images (which we call \"vokens\"). The \"vokenizer\" is trained on relatively small image captioning datasets and we then apply it to generate vokens for large language corpora. Trained with these contextually generated vokens, our visually-supervised language models show consistent improvements over self-supervised alternatives on multiple pure-language tasks such as GLUE, SQuAD, and SWAG. Code and pre-trained models publicly available at https://github.com/airsplay/vokenization","url_abs":"https://arxiv.org/abs/2010.06775v1","url_pdf":"https://arxiv.org/pdf/2010.06775v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"vokenization-improving-language-understanding","repo_url":"https://github.com/airsplay/vokenization","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"image-captioning","task_name":"Image Captioning"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"}],"methods":[{"method_slug":"vokenization","method_name":"Vokenization"}],"datasets_introduced":[],"methods_introduced":[{"slug":"vokenization","name":"Vokenization","full_name":"Vokenization"}],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2010.06775","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2010.06775"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/airsplay/vokenization","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":7},"by_repo_kind":{"official":{"samples":7,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"9cae12ce7150e792","entry":"IoU","repo":"airsplay/vokenization","repo_kind":"official","path":"vokenization/revokenization.py","file_url":"https://github.com/airsplay/vokenization/blob/HEAD/vokenization/revokenization.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"9cae12ce7150e792"}},{"code_sha256_prefix":"eb8574f8f0548a36","entry":"batchwise_hinge_rank_loss","repo":"airsplay/vokenization","repo_kind":"official","path":"xmatching/loss.py","file_url":"https://github.com/airsplay/vokenization/blob/HEAD/xmatching/loss.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"eb8574f8f0548a36"}},{"code_sha256_prefix":"198c3eb3f15298ae","entry":"get_visn_arch","repo":"airsplay/vokenization","repo_kind":"official","path":"vokenization/extract_vision_keys.py","file_url":"https://github.com/airsplay/vokenization/blob/HEAD/vokenization/extract_vision_keys.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"198c3eb3f15298ae"}},{"code_sha256_prefix":"0023db7e0b8b070b","entry":"hinge","repo":"airsplay/vokenization","repo_kind":"official","path":"xmatching/loss.py","file_url":"https://github.com/airsplay/vokenization/blob/HEAD/xmatching/loss.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"0023db7e0b8b070b"}},{"code_sha256_prefix":"8881c7f51b9a6a1f","entry":"load_lang_data","repo":"airsplay/vokenization","repo_kind":"official","path":"vokenization/evaluate_diversity.py","file_url":"https://github.com/airsplay/vokenization/blob/HEAD/vokenization/evaluate_diversity.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"8881c7f51b9a6a1f"}},{"code_sha256_prefix":"e32bf7939507dabe","entry":"memory_safe_apply","repo":"airsplay/vokenization","repo_kind":"official","path":"vokenization/vokenization.py","file_url":"https://github.com/airsplay/vokenization/blob/HEAD/vokenization/vokenization.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e32bf7939507dabe"}},{"code_sha256_prefix":"6e48e9b94ef67c44","entry":"paired_hinge_rank_loss","repo":"airsplay/vokenization","repo_kind":"official","path":"xmatching/loss.py","file_url":"https://github.com/airsplay/vokenization/blob/HEAD/xmatching/loss.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6e48e9b94ef67c44"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}