{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/on-the-downstream-performance-of-compressed","title":"On the Downstream Performance of Compressed Word Embeddings","arxiv_id":"1909.01264","date":"2019-09-03","proceeding":"NeurIPS 2019 12","authors":["Avner May","Jian Zhang","Tri Dao","Christopher Ré"],"abstract":"Compressing word embeddings is important for deploying NLP models in memory-constrained settings. However, understanding what makes compressed embeddings perform well on downstream tasks is challenging---existing measures of compression quality often fail to distinguish between embeddings that perform well and those that do not. We thus propose the eigenspace overlap score as a new measure. We relate the eigenspace overlap score to downstream performance by developing generalization bounds for the compressed embeddings in terms of this score, in the context of linear and logistic regression. We then show that we can lower bound the eigenspace overlap score for a simple uniform quantization compression method, helping to explain the strong empirical performance of this method. Finally, we show that by using the eigenspace overlap score as a selection criterion between embeddings drawn from a representative set we compressed, we can efficiently identify the better performing embedding with up to $2\\times$ lower selection error rates than the next best measure of compression quality, and avoid the cost of training a model for each task of interest.","url_abs":"https://arxiv.org/abs/1909.01264v2","url_pdf":"https://arxiv.org/pdf/1909.01264v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"on-the-downstream-performance-of-compressed","repo_url":"https://github.com/HazyResearch/smallfry","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"generalization-bounds","task_name":"Generalization Bounds"},{"task_slug":"quantization","task_name":"Quantization"},{"task_slug":"word-embeddings","task_name":"Word Embeddings"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/1909.01264","atlas_url":"https://app.syntology.ai/?focus=1909.01264","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"1909.01264"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/HazyResearch/smallfry","reach":null}],"summary":{"ran_fixture":2,"unverified":3},"by_repo_kind":{"official":{"samples":5,"ran":2,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"7fbe6134eb036758","entry":"compress_long_mat","repo":"HazyResearch/smallfry","repo_kind":"official","path":"src/smallfry/quant_embedding.py","file_url":"https://github.com/HazyResearch/smallfry/blob/HEAD/src/smallfry/quant_embedding.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"7fbe6134eb036758"}},{"code_sha256_prefix":"0a3f49fe8de06b20","entry":"decompress_long_mat","repo":"HazyResearch/smallfry","repo_kind":"official","path":"src/smallfry/quant_embedding.py","file_url":"https://github.com/HazyResearch/smallfry/blob/HEAD/src/smallfry/quant_embedding.py","link_basis":"first_harvest_node","language":"python","status":"ran_fixture","verification_level":1,"contract_check":"RAISES","metamorphic_tier":"invariant","behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"0a3f49fe8de06b20"}},{"code_sha256_prefix":"56ca64877b14f624","entry":"compress_and_compute_frob_squared_error","repo":"HazyResearch/smallfry","repo_kind":"official","path":"src/smallfry/compress.py","file_url":"https://github.com/HazyResearch/smallfry/blob/HEAD/src/smallfry/compress.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"56ca64877b14f624"}},{"code_sha256_prefix":"1cc1ad513150315f","entry":"compress_uniform","repo":"HazyResearch/smallfry","repo_kind":"official","path":"src/smallfry/compress.py","file_url":"https://github.com/HazyResearch/smallfry/blob/HEAD/src/smallfry/compress.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"1cc1ad513150315f"}},{"code_sha256_prefix":"2f36b0864e68d40c","entry":"find_optimal_range","repo":"HazyResearch/smallfry","repo_kind":"official","path":"src/smallfry/compress.py","file_url":"https://github.com/HazyResearch/smallfry/blob/HEAD/src/smallfry/compress.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"2f36b0864e68d40c"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}