{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/static-word-embeddings-for-sentence-semantic","title":"Static Word Embeddings for Sentence Semantic Representation","arxiv_id":"2506.04624","date":"2025-06-05","proceeding":null,"authors":["Takashi Wada","Yuki Hirakawa","Ryotaro Shimizu","Takahiro Kawashima","Yuki Saito"],"abstract":"We propose new static word embeddings optimised for sentence semantic representation. We first extract word embeddings from a pre-trained Sentence Transformer, and improve them with sentence-level principal component analysis, followed by either knowledge distillation or contrastive learning. During inference, we represent sentences by simply averaging word embeddings, which requires little computational cost. We evaluate models on both monolingual and cross-lingual tasks and show that our model substantially outperforms existing static models on sentence semantic tasks, and even rivals a basic Sentence Transformer model (SimCSE) on some data sets. Lastly, we perform a variety of analyses and show that our method successfully removes word embedding components that are irrelevant to sentence semantics, and adjusts the vector norms based on the influence of words on sentence semantics.","url_abs":"https://arxiv.org/abs/2506.04624v1","url_pdf":"https://arxiv.org/pdf/2506.04624v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[],"tasks":[{"task_slug":"contrastive-learning","task_name":"Contrastive Learning"},{"task_slug":"knowledge-distillation","task_name":"Knowledge Distillation"},{"task_slug":"sentence","task_name":"Sentence"},{"task_slug":"word-embeddings","task_name":"Word Embeddings"}],"methods":[{"method_slug":"absolute-position-encodings","method_name":"Absolute Position Encodings"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"knowledge-distillation","method_name":"Knowledge Distillation"},{"method_slug":"label-smoothing","method_name":"Label Smoothing"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"transformer","method_name":"Transformer"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2506.04624","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2506.04624"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"deterministic:regex_extraction","url":"https://github.com/twadada/swe4semantics","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/dleemiller/WordLlama","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"deterministic:regex_extraction","url":"https://github.com/MinishLab/model2vec","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_honours":1,"unverified":21},"by_repo_kind":{"found_in_text":{"samples":22,"ran":1,"repositories":3}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"09cf7bf4ad177244","entry":"approximate_function","repo":"dleemiller/WordLlama","repo_kind":"found_in_text","path":"src/wordllama/adapters/binarizer.py","file_url":"https://github.com/dleemiller/WordLlama/blob/HEAD/src/wordllama/adapters/binarizer.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"09cf7bf4ad177244"}},{"code_sha256_prefix":"f977b771831e8cfd","entry":"apply_pca","repo":"MinishLab/model2vec","repo_kind":"found_in_text","path":"model2vec/distill/inference.py","file_url":"https://github.com/MinishLab/model2vec/blob/HEAD/model2vec/distill/inference.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f977b771831e8cfd"}},{"code_sha256_prefix":"58271a558cadd0d5","entry":"binary_only","repo":"dleemiller/WordLlama","repo_kind":"found_in_text","path":"src/wordllama/mode_decorators.py","file_url":"https://github.com/dleemiller/WordLlama/blob/HEAD/src/wordllama/mode_decorators.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"58271a558cadd0d5"}},{"code_sha256_prefix":"69e02678a3940a6f","entry":"collect_sentences","repo":"twadada/swe4semantics","repo_kind":"found_in_text","path":"apply_pca_xling.py","file_url":"https://github.com/twadada/swe4semantics/blob/HEAD/apply_pca_xling.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"69e02678a3940a6f"}},{"code_sha256_prefix":"c0e21d259c6de207","entry":"compute_weights","repo":"MinishLab/model2vec","repo_kind":"found_in_text","path":"model2vec/distill/inference.py","file_url":"https://github.com/MinishLab/model2vec/blob/HEAD/model2vec/distill/inference.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"c0e21d259c6de207"}},{"code_sha256_prefix":"bc033863eb7b03a6","entry":"contrastive_loss","repo":"twadada/swe4semantics","repo_kind":"found_in_text","path":"train_xling.py","file_url":"https://github.com/twadada/swe4semantics/blob/HEAD/train_xling.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"bc033863eb7b03a6"}},{"code_sha256_prefix":"f50a4fe67500f002","entry":"create_embeddings","repo":"MinishLab/model2vec","repo_kind":"found_in_text","path":"model2vec/distill/inference.py","file_url":"https://github.com/MinishLab/model2vec/blob/HEAD/model2vec/distill/inference.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"f50a4fe67500f002"}},{"code_sha256_prefix":"dd25f90bf6f229e8","entry":"dense_only","repo":"dleemiller/WordLlama","repo_kind":"found_in_text","path":"src/wordllama/mode_decorators.py","file_url":"https://github.com/dleemiller/WordLlama/blob/HEAD/src/wordllama/mode_decorators.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"dd25f90bf6f229e8"}},{"code_sha256_prefix":"5ef8c214726b2b02","entry":"encode_batch_with_padding","repo":"twadada/swe4semantics","repo_kind":"found_in_text","path":"extract_embs.py","file_url":"https://github.com/twadada/swe4semantics/blob/HEAD/extract_embs.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"5ef8c214726b2b02"}},{"code_sha256_prefix":"3f0af3ab0ecd4794","entry":"encode_bilingual","repo":"twadada/swe4semantics","repo_kind":"found_in_text","path":"apply_pca_xling.py","file_url":"https://github.com/twadada/swe4semantics/blob/HEAD/apply_pca_xling.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"3f0af3ab0ecd4794"}},{"code_sha256_prefix":"4a593281de8cc870","entry":"encode_sentences","repo":"twadada/swe4semantics","repo_kind":"found_in_text","path":"apply_pca.py","file_url":"https://github.com/twadada/swe4semantics/blob/HEAD/apply_pca.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4a593281de8cc870"}},{"code_sha256_prefix":"51357003448ec154","entry":"encode_text","repo":"twadada/swe4semantics","repo_kind":"found_in_text","path":"util.py","file_url":"https://github.com/twadada/swe4semantics/blob/HEAD/util.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"51357003448ec154"}},{"code_sha256_prefix":"c5b4dcb8d8305191","entry":"encode_text_xling","repo":"twadada/swe4semantics","repo_kind":"found_in_text","path":"util.py","file_url":"https://github.com/twadada/swe4semantics/blob/HEAD/util.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c5b4dcb8d8305191"}},{"code_sha256_prefix":"916dbf02798a730a","entry":"evaluate_single_or_multi_label","repo":"MinishLab/model2vec","repo_kind":"found_in_text","path":"model2vec/inference/evaluation.py","file_url":"https://github.com/MinishLab/model2vec/blob/HEAD/model2vec/inference/evaluation.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"916dbf02798a730a"}},{"code_sha256_prefix":"a9c6c44e3bbf7f00","entry":"identify_word_indices","repo":"twadada/swe4semantics","repo_kind":"found_in_text","path":"extract_embs.py","file_url":"https://github.com/twadada/swe4semantics/blob/HEAD/extract_embs.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a9c6c44e3bbf7f00"}},{"code_sha256_prefix":"8f3ae471496c40f1","entry":"load_w2v","repo":"twadada/swe4semantics","repo_kind":"found_in_text","path":"util.py","file_url":"https://github.com/twadada/swe4semantics/blob/HEAD/util.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8f3ae471496c40f1"}},{"code_sha256_prefix":"2dd0e5f64e28e235","entry":"merge_word_vectors","repo":"twadada/swe4semantics","repo_kind":"found_in_text","path":"apply_pca_xling.py","file_url":"https://github.com/twadada/swe4semantics/blob/HEAD/apply_pca_xling.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2dd0e5f64e28e235"}},{"code_sha256_prefix":"a61103d9c8d1b43e","entry":"quantize_and_reduce_dim","repo":"MinishLab/model2vec","repo_kind":"found_in_text","path":"model2vec/quantization.py","file_url":"https://github.com/MinishLab/model2vec/blob/HEAD/model2vec/quantization.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a61103d9c8d1b43e"}},{"code_sha256_prefix":"6062e9d596058d89","entry":"quantize_embeddings","repo":"MinishLab/model2vec","repo_kind":"found_in_text","path":"model2vec/quantization.py","file_url":"https://github.com/MinishLab/model2vec/blob/HEAD/model2vec/quantization.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6062e9d596058d89"}},{"code_sha256_prefix":"27d601dc73cf7436","entry":"quantize_vocabulary","repo":"MinishLab/model2vec","repo_kind":"found_in_text","path":"model2vec/vocabulary_quantization.py","file_url":"https://github.com/MinishLab/model2vec/blob/HEAD/model2vec/vocabulary_quantization.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"27d601dc73cf7436"}},{"code_sha256_prefix":"14c9bef631548267","entry":"read_existing_vocabulary","repo":"twadada/swe4semantics","repo_kind":"found_in_text","path":"extract_embs.py","file_url":"https://github.com/twadada/swe4semantics/blob/HEAD/extract_embs.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"14c9bef631548267"}},{"code_sha256_prefix":"951c3eaccb80dafe","entry":"select_optimal_device","repo":"MinishLab/model2vec","repo_kind":"found_in_text","path":"model2vec/distill/utils.py","file_url":"https://github.com/MinishLab/model2vec/blob/HEAD/model2vec/distill/utils.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"951c3eaccb80dafe"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}