{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/training-sparse-mixture-of-experts-text","title":"Training Sparse Mixture Of Experts Text Embedding Models","arxiv_id":"2502.07972","date":"2025-02-11","proceeding":null,"authors":["Zach Nussbaum","Brandon Duderstadt"],"abstract":"Transformer-based text embedding models have improved their performance on benchmarks like MIRACL and BEIR by increasing their parameter counts. However, this scaling approach introduces significant deployment challenges, including increased inference latency and memory usage. These challenges are particularly severe in retrieval-augmented generation (RAG) applications, where large models' increased memory requirements constrain dataset ingestion capacity, and their higher latency directly impacts query-time performance. While causal language models have addressed similar efficiency challenges using Mixture of Experts (MoE) architectures, this approach hasn't been successfully adapted to the general text embedding setting. In this paper, we introduce Nomic Embed v2, the first general purpose MoE text embedding model. Our model outperforms models in the same parameter class on both monolingual and multilingual benchmarks while also maintaining competitive performance with models twice its size. We open-source all code, models, and evaluation data to ensure full reproducibility of our training pipeline at \\href{https://github.com/nomic-ai/contrastors}{https://github.com/nomic-ai/contrastors}.","url_abs":"https://arxiv.org/abs/2502.07972v3","url_pdf":"https://arxiv.org/pdf/2502.07972v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"training-sparse-mixture-of-experts-text","repo_url":"https://github.com/nomic-ai/contrastors","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}}],"tasks":[{"task_slug":"mixture-of-experts","task_name":"Mixture-of-Experts"},{"task_slug":"rag","task_name":"RAG"},{"task_slug":"retrieval-augmented-generation","task_name":"Retrieval-augmented Generation"}],"methods":[{"method_slug":"moe","method_name":"MoE"}],"datasets_introduced":[],"methods_introduced":[],"results":[],"syntology":{"syntology_url":null,"atlas_url":"https://app.syntology.ai/?focus=2502.07972","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2502.07972"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/nomic-ai/contrastors","reach":{"status":"ok","spdx":"Apache-2.0"}}],"summary":{"ran":1,"unverified":8},"by_repo_kind":{"official":{"samples":9,"ran":1,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"be4e00be50540042","entry":"log_and_continue","repo":"nomic-ai/contrastors","repo_kind":"official","path":"src/contrastors/dataset/image_text_loader.py","file_url":"https://github.com/nomic-ai/contrastors/blob/HEAD/src/contrastors/dataset/image_text_loader.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"be4e00be50540042"}},{"code_sha256_prefix":"a9ecfab3d0116215","entry":"calculate_auxiliary_loss","repo":"nomic-ai/contrastors","repo_kind":"official","path":"src/contrastors/loss.py","file_url":"https://github.com/nomic-ai/contrastors/blob/HEAD/src/contrastors/loss.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"a9ecfab3d0116215"}},{"code_sha256_prefix":"801ebbc0eea48637","entry":"collate_fn","repo":"nomic-ai/contrastors","repo_kind":"official","path":"src/contrastors/dataset/text_text_loader.py","file_url":"https://github.com/nomic-ai/contrastors/blob/HEAD/src/contrastors/dataset/text_text_loader.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"801ebbc0eea48637"}},{"code_sha256_prefix":"c522aff126ea966e","entry":"collate_local_ds","repo":"nomic-ai/contrastors","repo_kind":"official","path":"src/contrastors/dataset/text_text_loader.py","file_url":"https://github.com/nomic-ai/contrastors/blob/HEAD/src/contrastors/dataset/text_text_loader.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c522aff126ea966e"}},{"code_sha256_prefix":"6651be3ec3f1b0ce","entry":"configure_optimizer","repo":"nomic-ai/contrastors","repo_kind":"official","path":"src/contrastors/optimizer.py","file_url":"https://github.com/nomic-ai/contrastors/blob/HEAD/src/contrastors/optimizer.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"6651be3ec3f1b0ce"}},{"code_sha256_prefix":"e3e85af21ad96887","entry":"cutoff_long_text_for_embedding_generation","repo":"nomic-ai/contrastors","repo_kind":"official","path":"src/contrastors/eval/encoder.py","file_url":"https://github.com/nomic-ai/contrastors/blob/HEAD/src/contrastors/eval/encoder.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"e3e85af21ad96887"}},{"code_sha256_prefix":"8cea512acd127521","entry":"gather","repo":"nomic-ai/contrastors","repo_kind":"official","path":"src/contrastors/distributed.py","file_url":"https://github.com/nomic-ai/contrastors/blob/HEAD/src/contrastors/distributed.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"8cea512acd127521"}},{"code_sha256_prefix":"c54c334ddaf65c44","entry":"gather_dict","repo":"nomic-ai/contrastors","repo_kind":"official","path":"src/contrastors/distributed.py","file_url":"https://github.com/nomic-ai/contrastors/blob/HEAD/src/contrastors/distributed.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"c54c334ddaf65c44"}},{"code_sha256_prefix":"d02a4e692567ae9d","entry":"gather_with_grad","repo":"nomic-ai/contrastors","repo_kind":"official","path":"src/contrastors/distributed.py","file_url":"https://github.com/nomic-ai/contrastors/blob/HEAD/src/contrastors/distributed.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"d02a4e692567ae9d"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}