{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/sgpt-gpt-sentence-embeddings-for-semantic","title":"SGPT: GPT Sentence Embeddings for Semantic Search","arxiv_id":"2202.08904","date":"2022-02-17","proceeding":null,"authors":["Niklas Muennighoff"],"abstract":"Decoder transformers have continued increasing in scale reaching hundreds of billions of parameters. Due to their scale the same decoder sets state-of-the-art results on various language tasks via prompting or fine-tuning. Yet, these large foundation models remain unusable for the related fields of semantic search and sentence embeddings. This prevents possibly new state-of-the-art results and forces organizations to train and maintain separate models. To this end, we propose SGPT to use decoders for sentence embeddings and semantic search via prompting or fine-tuning. At 5.8 billion parameters SGPT improves on the previously best sentence embeddings by a margin of 7% and outperforms a concurrent method with 175 billion parameters as measured on the BEIR search benchmark. Code, models and result files are freely available at https://github.com/Muennighoff/sgpt.","url_abs":"https://arxiv.org/abs/2202.08904v5","url_pdf":"https://arxiv.org/pdf/2202.08904v5.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"sgpt-gpt-sentence-embeddings-for-semantic","repo_url":"https://github.com/muennighoff/sgpt","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"jax","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"argument-retrieval","task_name":"Argument Retrieval"},{"task_slug":"biomedical-information-retrieval","task_name":"Biomedical Information Retrieval"},{"task_slug":"citation-prediction","task_name":"Citation Prediction"},{"task_slug":"decoder","task_name":"Decoder"},{"task_slug":"duplicate-question-retrieval","task_name":"Duplicate-Question Retrieval"},{"task_slug":"entity-retrieval","task_name":"Entity Retrieval"},{"task_slug":"fact-checking","task_name":"Fact Checking"},{"task_slug":"information-retrieval","task_name":"Information Retrieval"},{"task_slug":"news-retrieval","task_name":"News Retrieval"},{"task_slug":"passage-retrieval","task_name":"Passage Retrieval"},{"task_slug":"question-answering","task_name":"Question Answering"},{"task_slug":"sentence","task_name":"Sentence"},{"task_slug":"sentence-embeddings","task_name":"Sentence Embeddings"},{"task_slug":"tweet-retrieval","task_name":"Tweet Retrieval"},{"task_slug":"zero-shot-text-search","task_name":"Zero-shot Text Search"}],"methods":[{"method_slug":"adam","method_name":"Adam"},{"method_slug":"attention","method_name":"Attention"},{"method_slug":"attention-dropout","method_name":"Attention Dropout"},{"method_slug":"bert","method_name":"BERT"},{"method_slug":"bpe","method_name":"BPE"},{"method_slug":"cosine-annealing","method_name":"Cosine Annealing"},{"method_slug":"dense-connections","method_name":"Dense Connections"},{"method_slug":"discriminative-fine-tuning","method_name":"Discriminative Fine-Tuning"},{"method_slug":"dropout","method_name":"Dropout"},{"method_slug":"gpt","method_name":"GPT"},{"method_slug":"layer-normalization","method_name":"Layer Normalization"},{"method_slug":"linear-layer","method_name":"Linear Layer"},{"method_slug":"linear-warmup-with-cosine-annealing","method_name":"Linear Warmup With Cosine Annealing"},{"method_slug":"linear-warmup-with-linear-decay","method_name":"Linear Warmup With Linear Decay"},{"method_slug":"multi-head-attention","method_name":"Multi-Head Attention"},{"method_slug":"residual-connection","method_name":"Residual Connection"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"weight-decay","method_name":"Weight Decay"},{"method_slug":"wordpiece","method_name":"WordPiece"}],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/biomedical-information-retrieval-on-bioasq-1","task":"Biomedical Information Retrieval","dataset":"BioASQ (BEIR)","model":"SGPT-CE-6.1B","rank_in_archive_order":2,"of":6,"metrics":{"nDCG@10":"0.547"},"uses_additional_data":false},{"leaderboard":"/sota/biomedical-information-retrieval-on-bioasq-1","task":"Biomedical Information Retrieval","dataset":"BioASQ (BEIR)","model":"SGPT-CE-2.7B","rank_in_archive_order":3,"of":6,"metrics":{"nDCG@10":" 0.546"},"uses_additional_data":false},{"leaderboard":"/sota/biomedical-information-retrieval-on-bioasq-1","task":"Biomedical Information Retrieval","dataset":"BioASQ (BEIR)","model":"SGPT-BE-5.8B","rank_in_archive_order":6,"of":6,"metrics":{"nDCG@10":"0.413"},"uses_additional_data":false},{"leaderboard":"/sota/biomedical-information-retrieval-on-nfcorpus-1","task":"Biomedical Information Retrieval","dataset":"NFCorpus (BEIR)","model":"SGPT-BE-5.8B","rank_in_archive_order":2,"of":7,"metrics":{"nDCG@10":"0.362"},"uses_additional_data":false},{"leaderboard":"/sota/biomedical-information-retrieval-on-nfcorpus-1","task":"Biomedical Information Retrieval","dataset":"NFCorpus (BEIR)","model":"OpenAI Search-Davinci","rank_in_archive_order":3,"of":7,"metrics":{"nDCG@10":"0.358"},"uses_additional_data":false},{"leaderboard":"/sota/biomedical-information-retrieval-on-nfcorpus-1","task":"Biomedical Information Retrieval","dataset":"NFCorpus (BEIR)","model":"SGPT-CE-6.1B","rank_in_archive_order":5,"of":7,"metrics":{"nDCG@10":" 0.347"},"uses_additional_data":false},{"leaderboard":"/sota/biomedical-information-retrieval-on-nfcorpus-1","task":"Biomedical Information Retrieval","dataset":"NFCorpus (BEIR)","model":"SGPT-CE-2.7B","rank_in_archive_order":6,"of":7,"metrics":{"nDCG@10":"0.333"},"uses_additional_data":false},{"leaderboard":"/sota/biomedical-information-retrieval-on-trec-1","task":"Biomedical Information Retrieval","dataset":"TREC-COVID (BEIR)","model":"SGPT-BE-5.8B","rank_in_archive_order":1,"of":6,"metrics":{"nDCG@10":"0.873"},"uses_additional_data":false},{"leaderboard":"/sota/biomedical-information-retrieval-on-trec-1","task":"Biomedical Information Retrieval","dataset":"TREC-COVID (BEIR)","model":"SGPT-CE-6.1B","rank_in_archive_order":3,"of":6,"metrics":{"nDCG@10":"0.791"},"uses_additional_data":false},{"leaderboard":"/sota/biomedical-information-retrieval-on-trec-1","task":"Biomedical Information Retrieval","dataset":"TREC-COVID (BEIR)","model":"SGPT-CE-2.7B","rank_in_archive_order":4,"of":6,"metrics":{"nDCG@10":"0.762"},"uses_additional_data":false},{"leaderboard":"/sota/fact-checking-on-climate-fever-beir","task":"Fact Checking","dataset":"CLIMATE-FEVER (BEIR)","model":"SGPT-BE-5.8B","rank_in_archive_order":1,"of":4,"metrics":{"nDCG@10":"0.305"},"uses_additional_data":false},{"leaderboard":"/sota/fact-checking-on-climate-fever-beir","task":"Fact Checking","dataset":"CLIMATE-FEVER (BEIR)","model":"SGPT-CE-6.1B","rank_in_archive_order":4,"of":4,"metrics":{"nDCG@10":" 0.161"},"uses_additional_data":false},{"leaderboard":"/sota/fact-checking-on-fever-beir","task":"Fact Checking","dataset":"FEVER (BEIR)","model":"SGPT-BE-5.8B","rank_in_archive_order":3,"of":4,"metrics":{"nDCG@10":"0.783"},"uses_additional_data":false},{"leaderboard":"/sota/fact-checking-on-fever-beir","task":"Fact Checking","dataset":"FEVER (BEIR)","model":"SGPT-CE-6.1B","rank_in_archive_order":4,"of":4,"metrics":{"nDCG@10":" 0.725"},"uses_additional_data":false},{"leaderboard":"/sota/fact-checking-on-scifact-beir","task":"Fact Checking","dataset":"SciFact (BEIR)","model":"SGPT-BE-5.8B","rank_in_archive_order":2,"of":5,"metrics":{"nDCG@10":"0.747"},"uses_additional_data":false},{"leaderboard":"/sota/fact-checking-on-scifact-beir","task":"Fact Checking","dataset":"SciFact (BEIR)","model":"SGPT-CE-6.1B","rank_in_archive_order":4,"of":5,"metrics":{"nDCG@10":"0.682"},"uses_additional_data":false},{"leaderboard":"/sota/information-retrieval-on-cqadupstack","task":"Information Retrieval","dataset":"CQADupStack","model":"SGPT-BE-5.8B","rank_in_archive_order":1,"of":2,"metrics":{"mAP@100":"0.160"},"uses_additional_data":false},{"leaderboard":"/sota/passage-retrieval-on-msmarco-beir","task":"Passage Retrieval","dataset":"MSMARCO (BEIR)","model":"SGPT-BE-5.8B","rank_in_archive_order":4,"of":12,"metrics":{"nDCG@10":"0.399"},"uses_additional_data":false},{"leaderboard":"/sota/passage-retrieval-on-msmarco-beir","task":"Passage Retrieval","dataset":"MSMARCO (BEIR)","model":"SGPT-CE-6.1B","rank_in_archive_order":9,"of":12,"metrics":{"nDCG@10":"0.290"},"uses_additional_data":false},{"leaderboard":"/sota/passage-retrieval-on-msmarco-beir","task":"Passage Retrieval","dataset":"MSMARCO (BEIR)","model":"SGPT-CE-2.7B","rank_in_archive_order":10,"of":12,"metrics":{"nDCG@10":"0.278"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-fiqa-2018-beir","task":"Question Answering","dataset":"FiQA-2018 (BEIR)","model":"SGPT-CE-6.1B","rank_in_archive_order":2,"of":4,"metrics":{"nDCG@10":"0.401"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-fiqa-2018-beir","task":"Question Answering","dataset":"FiQA-2018 (BEIR)","model":"SGPT-BE-5.8B","rank_in_archive_order":3,"of":4,"metrics":{"nDCG@10":"0.372"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-hotpotqa-beir","task":"Question Answering","dataset":"HotpotQA (BEIR)","model":"SGPT-CE-6.1B","rank_in_archive_order":3,"of":4,"metrics":{"nDCG@10":" 0.699"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-hotpotqa-beir","task":"Question Answering","dataset":"HotpotQA (BEIR)","model":"SGPT-BE-5.8B","rank_in_archive_order":4,"of":4,"metrics":{"nDCG@10":"0.593"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-nq-beir","task":"Question Answering","dataset":"NQ (BEIR)","model":"SGPT-BE-5.8B","rank_in_archive_order":5,"of":6,"metrics":{"nDCG@10":"0.524"},"uses_additional_data":false},{"leaderboard":"/sota/question-answering-on-nq-beir","task":"Question Answering","dataset":"NQ (BEIR)","model":"SGPT-CE-6.1B","rank_in_archive_order":6,"of":6,"metrics":{"nDCG@10":" 0.401"},"uses_additional_data":false}],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2202.08904","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2202.08904"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/muennighoff/sgpt","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"ran_draft_wrong":1},"by_repo_kind":{"official":{"samples":1,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"51678dd4874a6563","entry":"clean_titles","repo":"muennighoff/sgpt","repo_kind":"official","path":"biencoder/beir/beir_dense_retriever.py","file_url":"https://github.com/muennighoff/sgpt/blob/HEAD/biencoder/beir/beir_dense_retriever.py","link_basis":"first_harvest_node","language":"python","status":"ran_draft_wrong","verification_level":1,"contract_check":"OUTPUT_MISDECLARED","metamorphic_tier":"deterministic","behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"51678dd4874a6563"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}