{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/jasper-and-stella-distillation-of-sota","title":"Jasper and Stella: distillation of SOTA embedding models","arxiv_id":"2412.19048","date":"2024-12-26","proceeding":null,"authors":["Dun Zhang","Jiacheng Li","Ziyang Zeng","Fulong Wang"],"abstract":"A crucial component in many deep learning applications, such as Frequently Asked Questions (FAQ) and Retrieval-Augmented Generation (RAG), is dense retrieval. In this process, embedding models transform raw text into numerical vectors. However, the embedding models that currently excel on text embedding benchmarks, like the Massive Text Embedding Benchmark (MTEB), often have numerous parameters and high vector dimensionality. This poses challenges for their application in real-world scenarios. To address this issue, we propose a novel multi-stage distillation framework that enables a smaller student embedding model to distill multiple larger teacher embedding models through three carefully designed losses. Meanwhile, we utilize Matryoshka Representation Learning (MRL) to reduce the vector dimensionality of the student embedding model effectively. Our student model named Jasper with 2 billion parameters, built upon the Stella embedding model, obtained the No.3 position on the MTEB leaderboard (as of December 24, 2024), achieving an average 71.54 score across 56 datasets. We have released the model and data on the Hugging Face Hub (https://huggingface.co/infgrad/jasper_en_vision_language_v1) (https://huggingface.co/datasets/infgrad/jasper_text_distill_dataset), and the training codes are available in this project repository (https://github.com/NLPJCL/RAG-Retrieval).","url_abs":"https://arxiv.org/abs/2412.19048v2","url_pdf":"https://arxiv.org/pdf/2412.19048v2.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"jasper-and-stella-distillation-of-sota","repo_url":"https://github.com/NLPJCL/RAG-Retrieval","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}},{"paper_slug":"jasper-and-stella-distillation-of-sota","repo_url":"https://github.com/novasearch-team/rag-retrieval","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":null}],"tasks":[{"task_slug":"rag","task_name":"RAG"},{"task_slug":"representation-learning","task_name":"Representation Learning"},{"task_slug":"retrieval","task_name":"Retrieval"},{"task_slug":"retrieval-augmented-generation","task_name":"Retrieval-augmented Generation"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":"https://syntology.ai/paper/2412.19048","atlas_url":"https://app.syntology.ai/?focus=2412.19048","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2412.19048"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-25T09:33:49+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/NLPJCL/RAG-Retrieval","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/novasearch-team/rag-retrieval","reach":null}],"summary":{"ran_violates":1,"ran":3},"by_repo_kind":{"official":{"samples":4,"ran":4,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"b8e95809ca2c17c9","entry":"sigmoid","repo":"NLPJCL/RAG-Retrieval","repo_kind":"official","path":"rag_retrieval/infer/reranker_models/llm_rankers.py","file_url":"https://github.com/NLPJCL/RAG-Retrieval/blob/HEAD/rag_retrieval/infer/reranker_models/llm_rankers.py","link_basis":"harvester_set","language":"python","status":"ran_violates","verification_level":2,"contract_check":"VIOLATES","metamorphic_tier":"invariant","behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b8e95809ca2c17c9"}},{"code_sha256_prefix":"3e96bdcb68881aa5","entry":"pairwise_ranknet","repo":"NLPJCL/RAG-Retrieval","repo_kind":"official","path":"rag_retrieval/train/reranker/ranking_loss.py","file_url":"https://github.com/NLPJCL/RAG-Retrieval/blob/HEAD/rag_retrieval/train/reranker/ranking_loss.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"3e96bdcb68881aa5"}},{"code_sha256_prefix":"4be0550994765aff","entry":"pointwise_bce","repo":"NLPJCL/RAG-Retrieval","repo_kind":"official","path":"rag_retrieval/train/reranker/ranking_loss.py","file_url":"https://github.com/NLPJCL/RAG-Retrieval/blob/HEAD/rag_retrieval/train/reranker/ranking_loss.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"4be0550994765aff"}},{"code_sha256_prefix":"b07e6aba62f37f14","entry":"pointwise_mse","repo":"NLPJCL/RAG-Retrieval","repo_kind":"official","path":"rag_retrieval/train/reranker/ranking_loss.py","file_url":"https://github.com/NLPJCL/RAG-Retrieval/blob/HEAD/rag_retrieval/train/reranker/ranking_loss.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"b07e6aba62f37f14"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}