{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/multilingual-alignment-of-contextual-word-1","title":"Multilingual Alignment of Contextual Word Representations","arxiv_id":"2002.03518","date":"2020-02-10","proceeding":"ICLR 2020 1","authors":["Steven Cao","Nikita Kitaev","Dan Klein"],"abstract":"We propose procedures for evaluating and strengthening contextual embedding alignment and show that they are useful in analyzing and improving multilingual BERT. In particular, after our proposed alignment procedure, BERT exhibits significantly improved zero-shot performance on XNLI compared to the base model, remarkably matching pseudo-fully-supervised translate-train models for Bulgarian and Greek. Further, to measure the degree of alignment, we introduce a contextual version of word retrieval and show that it correlates well with downstream zero-shot transfer. Using this word retrieval task, we also analyze BERT and find that it exhibits systematic deficiencies, e.g. worse alignment for open-class parts-of-speech and word pairs written in different scripts, that are corrected by the alignment procedure. These results support contextual alignment as a useful concept for understanding large multilingual pre-trained models.","url_abs":"https://arxiv.org/abs/2002.03518v2","url_pdf":"https://arxiv.org/pdf/2002.03518v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"multilingual-alignment-of-contextual-word-1","repo_url":"https://github.com/pefimov/cross-lingual-adjustment","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"retrieval","task_name":"Retrieval"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2002.03518","atlas_url":"https://app.syntology.ai/?focus=2002.03518","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2002.03518"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/pefimov/cross-lingual-adjustment","reach":null}],"summary":{"ran_draft_wrong":1,"unverified":2},"by_repo_kind":{"listed":{"samples":3,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":3,"samples":[{"code_sha256_prefix":"64f51fd9dbad96dc","entry":"partial_sums","repo":"pefimov/cross-lingual-adjustment","repo_kind":"listed","path":"alignment/bert_finetuning_alignment.py","file_url":"https://github.com/pefimov/cross-lingual-adjustment/blob/HEAD/alignment/bert_finetuning_alignment.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"MISDECLARED","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"64f51fd9dbad96dc"}},{"code_sha256_prefix":"5acf98bead894abe","entry":"align_bert_multiple","repo":"pefimov/cross-lingual-adjustment","repo_kind":"listed","path":"alignment/bert_finetuning_alignment.py","file_url":"https://github.com/pefimov/cross-lingual-adjustment/blob/HEAD/alignment/bert_finetuning_alignment.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5acf98bead894abe"}},{"code_sha256_prefix":"4772dda96d99e52e","entry":"pick_aligned","repo":"pefimov/cross-lingual-adjustment","repo_kind":"listed","path":"alignment/bert_finetuning_alignment.py","file_url":"https://github.com/pefimov/cross-lingual-adjustment/blob/HEAD/alignment/bert_finetuning_alignment.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"4772dda96d99e52e"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}