{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/tsdae-using-transformer-based-sequential","title":"TSDAE: Using Transformer-based Sequential Denoising Auto-Encoder for Unsupervised Sentence Embedding Learning","arxiv_id":"2104.06979","date":"2021-04-14","proceeding":null,"authors":["Kexin Wang","Nils Reimers","Iryna Gurevych"],"abstract":"Learning sentence embeddings often requires a large amount of labeled data. However, for most tasks and domains, labeled data is seldom available and creating it is expensive. In this work, we present a new state-of-the-art unsupervised method based on pre-trained Transformers and Sequential Denoising Auto-Encoder (TSDAE) which outperforms previous approaches by up to 6.4 points. It can achieve up to 93.1% of the performance of in-domain supervised approaches. Further, we show that TSDAE is a strong domain adaptation and pre-training method for sentence embeddings, significantly outperforming other approaches like Masked Language Model. A crucial shortcoming of previous studies is the narrow evaluation: Most work mainly evaluates on the single task of Semantic Textual Similarity (STS), which does not require any domain knowledge. It is unclear if these proposed methods generalize to other domains and tasks. We fill this gap and evaluate TSDAE and other recent approaches on four different datasets from heterogeneous domains.","url_abs":"https://arxiv.org/abs/2104.06979v3","url_pdf":"https://arxiv.org/pdf/2104.06979v3.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"tsdae-using-transformer-based-sequential","repo_url":"https://github.com/kwang2049/pytorch-bertflow","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"tsdae-using-transformer-based-sequential","repo_url":"https://github.com/kwang2049/useb","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"tsdae-using-transformer-based-sequential","repo_url":"https://github.com/ukplab/pytorch-bertflow","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"tsdae-using-transformer-based-sequential","repo_url":"https://github.com/UKPLab/useb","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"Apache-2.0"}},{"paper_slug":"tsdae-using-transformer-based-sequential","repo_url":"https://github.com/climsocana/tecb-de","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"none","reach":null},{"paper_slug":"tsdae-using-transformer-based-sequential","repo_url":"https://github.com/exploration-lab/iitk-semeval-2024-task-1","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"denoising","task_name":"Denoising"},{"task_slug":"domain-adaptation","task_name":"Domain Adaptation"},{"task_slug":"information-retrieval","task_name":"Information Retrieval"},{"task_slug":"language-modeling","task_name":"Language Modeling"},{"task_slug":"language-modelling","task_name":"Language Modelling"},{"task_slug":"paraphrase-identification","task_name":"Paraphrase Identification"},{"task_slug":"re-ranking","task_name":"Re-Ranking"},{"task_slug":"sts","task_name":"STS"},{"task_slug":"semantic-textual-similarity","task_name":"Semantic Textual Similarity"},{"task_slug":"sentence","task_name":"Sentence"},{"task_slug":"sentence-embedding","task_name":"Sentence Embedding"},{"task_slug":"sentence-embeddings","task_name":"Sentence Embeddings"},{"task_slug":"sentence-embedding-1","task_name":"Sentence-Embedding"}],"methods":[{"method_slug":"attention","method_name":"Attention"},{"method_slug":"softmax","method_name":"Softmax"},{"method_slug":"tsdae","method_name":"TSDAE"}],"datasets_introduced":[],"methods_introduced":[{"slug":"tsdae","name":"TSDAE","full_name":"TSDAE"}],"results":[{"leaderboard":"/sota/information-retrieval-on-cqadupstack","task":"Information Retrieval","dataset":"CQADupStack","model":"TSDAE","rank_in_archive_order":2,"of":2,"metrics":{"mAP@100":"0.145"},"uses_additional_data":false},{"leaderboard":"/sota/paraphrase-identification-on-pit","task":"Paraphrase Identification","dataset":"PIT","model":"TSDAE","rank_in_archive_order":1,"of":1,"metrics":{"AP":"69.2"},"uses_additional_data":false},{"leaderboard":"/sota/paraphrase-identification-on-turl","task":"Paraphrase Identification","dataset":"TURL","model":"TSDAE","rank_in_archive_order":1,"of":1,"metrics":{"AP":"76.8"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2104.06979","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2104.06979"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/UKPLab/useb","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kwang2049/pytorch-bertflow","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/climsocana/tecb-de","reach":null},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/kwang2049/useb","reach":{"status":"ok","spdx":"Apache-2.0"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ukplab/pytorch-bertflow","reach":{"status":"ok"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/exploration-lab/iitk-semeval-2024-task-1","reach":{"status":"ok","spdx":"MIT"}}],"summary":{"unverified":4},"by_repo_kind":{"official":{"samples":4,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":0,"samples":[{"code_sha256_prefix":"98ef1f429528f0af","entry":"ap_score","repo":"kwang2049/useb","repo_kind":"official","path":"useb/evaluators/askubuntu.py","file_url":"https://github.com/kwang2049/useb/blob/HEAD/useb/evaluators/askubuntu.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"98ef1f429528f0af"}},{"code_sha256_prefix":"f8c5243b757f193d","entry":"rank_by_score","repo":"kwang2049/useb","repo_kind":"official","path":"useb/evaluators/askubuntu.py","file_url":"https://github.com/kwang2049/useb/blob/HEAD/useb/evaluators/askubuntu.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"f8c5243b757f193d"}},{"code_sha256_prefix":"4ea966908d8fd441","entry":"reciprocal_rank","repo":"kwang2049/useb","repo_kind":"official","path":"useb/evaluators/askubuntu.py","file_url":"https://github.com/kwang2049/useb/blob/HEAD/useb/evaluators/askubuntu.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4ea966908d8fd441"}},{"code_sha256_prefix":"4b4d63cfd3f3c390","entry":"run","repo":"kwang2049/useb","repo_kind":"official","path":"useb/evaluating.py","file_url":"https://github.com/kwang2049/useb/blob/HEAD/useb/evaluating.py","link_basis":"first_harvest_node","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"Apache-2.0","inline_ok":true,"mcp_get_code":{"code_sha256":"4b4d63cfd3f3c390"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}