{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/direction-is-what-you-need-improving-word","title":"Direction is what you need: Improving Word Embedding Compression in Large Language Models","arxiv_id":"2106.08181","date":"2021-06-15","proceeding":"ACL (RepL4NLP) 2021 8","authors":["Klaudia Bałazy","Mohammadreza Banaei","Rémi Lebret","Jacek Tabor","Karl Aberer"],"abstract":"The adoption of Transformer-based models in natural language processing (NLP) has led to great success using a massive number of parameters. However, due to deployment constraints in edge devices, there has been a rising interest in the compression of these models to improve their inference time and memory footprint. This paper presents a novel loss objective to compress token embeddings in the Transformer-based models by leveraging an AutoEncoder architecture. More specifically, we emphasize the importance of the direction of compressed embeddings with respect to original uncompressed embeddings. The proposed method is task-agnostic and does not require further language modeling pre-training. Our method significantly outperforms the commonly used SVD-based matrix-factorization approach in terms of initial language model Perplexity. Moreover, we evaluate our proposed approach over SQuAD v1.1 dataset and several downstream tasks from the GLUE benchmark, where we also outperform the baseline in most scenarios. Our code is public.","url_abs":"https://arxiv.org/abs/2106.08181v2","url_pdf":"https://arxiv.org/pdf/2106.08181v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"direction-is-what-you-need-improving-word","repo_url":"https://github.com/MohammadrezaBanaei/orientation_based_embedding_compression","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2106.08181","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2106.08181"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/MohammadrezaBanaei/orientation_based_embedding_compression","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_honours":1,"unverified":6},"by_repo_kind":{"official":{"samples":7,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"7337f1f5ff01dcd0","entry":"get_lr","repo":"MohammadrezaBanaei/orientation_based_embedding_compression","repo_kind":"official","path":"ae_train.py","file_url":"https://github.com/MohammadrezaBanaei/orientation_based_embedding_compression/blob/HEAD/ae_train.py","link_basis":"harvester_set","language":"python","status":"ran_honours","verification_level":1,"contract_check":"HONOURS","metamorphic_tier":"well_formed","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"7337f1f5ff01dcd0"}},{"code_sha256_prefix":"268611043c2b35f8","entry":"cosine_distance_loss","repo":"MohammadrezaBanaei/orientation_based_embedding_compression","repo_kind":"official","path":"losses/cosine_distance_loss.py","file_url":"https://github.com/MohammadrezaBanaei/orientation_based_embedding_compression/blob/HEAD/losses/cosine_distance_loss.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"268611043c2b35f8"}},{"code_sha256_prefix":"a92218ae0d90dd05","entry":"get_layer_params_num","repo":"MohammadrezaBanaei/orientation_based_embedding_compression","repo_kind":"official","path":"utils_funcs.py","file_url":"https://github.com/MohammadrezaBanaei/orientation_based_embedding_compression/blob/HEAD/utils_funcs.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a92218ae0d90dd05"}},{"code_sha256_prefix":"204006412d2334a0","entry":"get_model_weight_dict","repo":"MohammadrezaBanaei/orientation_based_embedding_compression","repo_kind":"official","path":"transformers_data_utils.py","file_url":"https://github.com/MohammadrezaBanaei/orientation_based_embedding_compression/blob/HEAD/transformers_data_utils.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"204006412d2334a0"}},{"code_sha256_prefix":"79649aea92c2caa0","entry":"l1_norm_loss","repo":"MohammadrezaBanaei/orientation_based_embedding_compression","repo_kind":"official","path":"losses/l1_norm_loss.py","file_url":"https://github.com/MohammadrezaBanaei/orientation_based_embedding_compression/blob/HEAD/losses/l1_norm_loss.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"79649aea92c2caa0"}},{"code_sha256_prefix":"25471ca70b09e529","entry":"l2_norm_loss","repo":"MohammadrezaBanaei/orientation_based_embedding_compression","repo_kind":"official","path":"losses/l2_norm_loss.py","file_url":"https://github.com/MohammadrezaBanaei/orientation_based_embedding_compression/blob/HEAD/losses/l2_norm_loss.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"25471ca70b09e529"}},{"code_sha256_prefix":"a3de5ce5addb03b3","entry":"run_svd","repo":"MohammadrezaBanaei/orientation_based_embedding_compression","repo_kind":"official","path":"utils_funcs.py","file_url":"https://github.com/MohammadrezaBanaei/orientation_based_embedding_compression/blob/HEAD/utils_funcs.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"a3de5ce5addb03b3"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}