{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/scaling-sentence-embeddings-with-large","title":"Scaling Sentence Embeddings with Large Language Models","arxiv_id":"2307.16645","date":"2023-07-31","proceeding":null,"authors":["Ting Jiang","Shaohan Huang","Zhongzhi Luan","Deqing Wang","Fuzhen Zhuang"],"abstract":"Large language models (LLMs) have recently garnered significant interest. With in-context learning, LLMs achieve impressive results in various natural language tasks. However, the application of LLMs to sentence embeddings remains an area of ongoing research. In this work, we propose an in-context learning-based method aimed at improving sentence embeddings performance. Our approach involves adapting the previous prompt-based representation method for autoregressive models, constructing a demonstration set that enables LLMs to perform in-context learning, and scaling up the LLMs to different model sizes. Through extensive experiments, in-context learning enables LLMs to generate high-quality sentence embeddings without any fine-tuning. It helps LLMs achieve performance comparable to current contrastive learning methods. By scaling model size, we find scaling to more than tens of billion parameters harms the performance on semantic textual similarity (STS) tasks. However, the largest model outperforms other counterparts and achieves the new state-of-the-art result on transfer tasks. We also fine-tune LLMs with current contrastive learning approach, and the 2.7B OPT model, incorporating our prompt-based method, surpasses the performance of 4.8B ST5, achieving the new state-of-the-art results on STS tasks. Our code is available at https://github.com/kongds/scaling_sentemb.","url_abs":"https://arxiv.org/abs/2307.16645v1","url_pdf":"https://arxiv.org/pdf/2307.16645v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"scaling-sentence-embeddings-with-large","repo_url":"https://github.com/kongds/scaling_sentemb","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok"}}],"tasks":[{"task_slug":"contrastive-learning","task_name":"Contrastive Learning"},{"task_slug":"in-context-learning","task_name":"In-Context Learning"},{"task_slug":"sts","task_name":"STS"},{"task_slug":"semantic-textual-similarity","task_name":"Semantic Textual Similarity"},{"task_slug":"sentence","task_name":"Sentence"},{"task_slug":"sentence-embedding","task_name":"Sentence Embedding"},{"task_slug":"sentence-embeddings","task_name":"Sentence Embeddings"}],"methods":[{"method_slug":"contrastive-learning","method_name":"Contrastive Learning"},{"method_slug":"llama","method_name":"LLaMA"},{"method_slug":"opt","method_name":"OPT"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/semantic-textual-similarity-on-sick","task":"Semantic Textual Similarity","dataset":"SICK","model":"PromptEOL+CSE+LLaMA-30B","rank_in_archive_order":2,"of":22,"metrics":{"Spearman Correlation":"0.8238"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-sick","task":"Semantic Textual Similarity","dataset":"SICK","model":"PromptEOL+CSE+OPT-13B","rank_in_archive_order":3,"of":22,"metrics":{"Spearman Correlation":"0.8206"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-sick","task":"Semantic Textual Similarity","dataset":"SICK","model":"PromptEOL+CSE+OPT-2.7B","rank_in_archive_order":5,"of":22,"metrics":{"Spearman Correlation":"0.8129"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-sts-benchmark","task":"Semantic Textual Similarity","dataset":"STS Benchmark","model":"PromptEOL+CSE+LLaMA-30B","rank_in_archive_order":32,"of":66,"metrics":{"Spearman Correlation":"0.8914"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-sts-benchmark","task":"Semantic Textual Similarity","dataset":"STS Benchmark","model":"PromptEOL+CSE+OPT-13B","rank_in_archive_order":36,"of":66,"metrics":{"Spearman Correlation":"0.8856"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-sts-benchmark","task":"Semantic Textual Similarity","dataset":"STS Benchmark","model":"PromptEOL+CSE+OPT-2.7B","rank_in_archive_order":37,"of":66,"metrics":{"Spearman Correlation":"0.8833"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-sts12","task":"Semantic Textual Similarity","dataset":"STS12","model":"PromptEOL+CSE+OPT-13B","rank_in_archive_order":1,"of":20,"metrics":{"Spearman Correlation":"0.8020"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-sts12","task":"Semantic Textual Similarity","dataset":"STS12","model":"PromptEOL+CSE+LLaMA-30B","rank_in_archive_order":2,"of":20,"metrics":{"Spearman Correlation":"0.7972"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-sts12","task":"Semantic Textual Similarity","dataset":"STS12","model":"PromptEOL+CSE+OPT-2.7B","rank_in_archive_order":4,"of":20,"metrics":{"Spearman Correlation":"0.7949"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-sts13","task":"Semantic Textual Similarity","dataset":"STS13","model":"PromptEOL+CSE+LLaMA-30B","rank_in_archive_order":3,"of":22,"metrics":{"Spearman Correlation":"0.9025"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-sts13","task":"Semantic Textual Similarity","dataset":"STS13","model":"PromptEOL+CSE+OPT-13B","rank_in_archive_order":4,"of":22,"metrics":{"Spearman Correlation":"0.9024"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-sts13","task":"Semantic Textual Similarity","dataset":"STS13","model":"PromptEOL+CSE+OPT-2.7B","rank_in_archive_order":5,"of":22,"metrics":{"Spearman Correlation":"0.8964"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-sts14","task":"Semantic Textual Similarity","dataset":"STS14","model":"PromptEOL+CSE+LLaMA-30B","rank_in_archive_order":2,"of":21,"metrics":{"Spearman Correlation":"0.8585"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-sts14","task":"Semantic Textual Similarity","dataset":"STS14","model":"PromptEOL+CSE+OPT-13B","rank_in_archive_order":5,"of":21,"metrics":{"Spearman Correlation":"0.8534"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-sts14","task":"Semantic Textual Similarity","dataset":"STS14","model":"PromptEOL+CSE+OPT-2.7B","rank_in_archive_order":6,"of":21,"metrics":{"Spearman Correlation":"0.8480"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-sts15","task":"Semantic Textual Similarity","dataset":"STS15","model":"PromptEOL+CSE+LLaMA-30B","rank_in_archive_order":1,"of":20,"metrics":{"Spearman Correlation":"0.9004"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-sts15","task":"Semantic Textual Similarity","dataset":"STS15","model":"PromptEOL+CSE+OPT-13B","rank_in_archive_order":3,"of":20,"metrics":{"Spearman Correlation":"0.8952"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-sts15","task":"Semantic Textual Similarity","dataset":"STS15","model":"PromptEOL+CSE+OPT-2.7B","rank_in_archive_order":4,"of":20,"metrics":{"Spearman Correlation":"0.8951"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-sts16","task":"Semantic Textual Similarity","dataset":"STS16","model":"PromptEOL+CSE+LLaMA-30B","rank_in_archive_order":4,"of":20,"metrics":{"Spearman Correlation":"0.8627"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-sts16","task":"Semantic Textual Similarity","dataset":"STS16","model":"PromptEOL+CSE+OPT-2.7B","rank_in_archive_order":5,"of":20,"metrics":{"Spearman Correlation":"0.8591"},"uses_additional_data":false},{"leaderboard":"/sota/semantic-textual-similarity-on-sts16","task":"Semantic Textual Similarity","dataset":"STS16","model":"PromptEOL+CSE+OPT-13B","rank_in_archive_order":6,"of":20,"metrics":{"Spearman Correlation":"0.8590"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2307.16645","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2307.16645"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kongds/scaling_sentemb","reach":{"status":"ok"}}],"summary":{"ran":4},"by_repo_kind":{"official":{"samples":4,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":4,"samples":[{"code_sha256_prefix":"ab968085774a0a8d","entry":"cal_avg_cosine","repo":"kongds/scaling_sentemb","repo_kind":"official","path":"evaluation_with_multi_templates.py","file_url":"https://github.com/kongds/scaling_sentemb/blob/HEAD/evaluation_with_multi_templates.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ab968085774a0a8d"}},{"code_sha256_prefix":"541562fdc4531cd9","entry":"generate_sentemb_prompt","repo":"kongds/scaling_sentemb","repo_kind":"official","path":"ft_llm.py","file_url":"https://github.com/kongds/scaling_sentemb/blob/HEAD/ft_llm.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"541562fdc4531cd9"}},{"code_sha256_prefix":"083d6c8d62e4a0d5","entry":"get_delta","repo":"kongds/scaling_sentemb","repo_kind":"official","path":"evaluation_with_multi_templates.py","file_url":"https://github.com/kongds/scaling_sentemb/blob/HEAD/evaluation_with_multi_templates.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"083d6c8d62e4a0d5"}},{"code_sha256_prefix":"0a116d23cfc3e51b","entry":"s_eval","repo":"kongds/scaling_sentemb","repo_kind":"official","path":"evaluation_with_multi_templates.py","file_url":"https://github.com/kongds/scaling_sentemb/blob/HEAD/evaluation_with_multi_templates.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"0a116d23cfc3e51b"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}