{"about":{"site":"https://codewithpapers.app","non_affiliation":"Code with Papers and Syntology are not affiliated with, endorsed by, or sponsored by Papers with Code, Meta, or the pwc-archive mirror.","licence":"CC BY-SA 4.0","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","attribution":"https://codewithpapers.app/attribution","modified":"archive material modified by Syntology; see the attribution page"},"url":"/paper/emotion2vec-self-supervised-pre-training-for","title":"emotion2vec: Self-Supervised Pre-Training for Speech Emotion Representation","arxiv_id":"2312.15185","date":"2023-12-23","proceeding":null,"authors":["Ziyang Ma","Zhisheng Zheng","Jiaxin Ye","Jinchao Li","Zhifu Gao","Shiliang Zhang","Xie Chen"],"abstract":"We propose emotion2vec, a universal speech emotion representation model. emotion2vec is pre-trained on open-source unlabeled emotion data through self-supervised online distillation, combining utterance-level loss and frame-level loss during pre-training. emotion2vec outperforms state-of-the-art pre-trained universal models and emotion specialist models by only training linear layers for the speech emotion recognition task on the mainstream IEMOCAP dataset. In addition, emotion2vec shows consistent improvements among 10 different languages of speech emotion recognition datasets. emotion2vec also shows excellent results on other emotion tasks, such as song emotion recognition, emotion prediction in conversation, and sentiment analysis. Comparison experiments, ablation experiments, and visualization comprehensively demonstrate the universal capability of the proposed emotion2vec. To the best of our knowledge, emotion2vec is the first universal representation model in various emotion-related tasks, filling a gap in the field.","url_abs":"https://arxiv.org/abs/2312.15185v1","url_pdf":"https://arxiv.org/pdf/2312.15185v1.pdf","source":{"archive":"pwc-archive (Hugging Face), CC BY-SA 4.0","snapshot":"2025-07-28","licence_url":"https://creativecommons.org/licenses/by-sa/4.0/legalcode","row_kind":"abstracts"},"code_links":[{"paper_slug":"emotion2vec-self-supervised-pre-training-for","repo_url":"https://github.com/ddlBoJack/emotion2vec","is_official":1,"mentioned_in_paper":1,"mentioned_in_github":1,"framework":"pytorch","reach":{"status":"ok"}},{"paper_slug":"emotion2vec-self-supervised-pre-training-for","repo_url":"https://github.com/alibaba-damo-academy/FunASR","is_official":0,"mentioned_in_paper":0,"mentioned_in_github":0,"framework":"pytorch","reach":{"status":"ok","spdx":"MIT"}}],"tasks":[{"task_slug":"emotion-recognition","task_name":"Emotion Recognition"},{"task_slug":"self-supervised-learning","task_name":"Self-Supervised Learning"},{"task_slug":"sentiment-analysis","task_name":"Sentiment Analysis"},{"task_slug":"speech-emotion-recognition","task_name":"Speech Emotion Recognition"}],"methods":[],"datasets_introduced":[],"methods_introduced":[],"results":[{"leaderboard":"/sota/speech-emotion-recognition-on-resd","task":"Speech Emotion Recognition","dataset":"RESD","model":"emotion2vec+base","rank_in_archive_order":1,"of":3,"metrics":{"Unweighted Accuracy (UA)":"79.8","Weighted Accuracy (WA)":"79.4","Weighted F1":"79.4"},"uses_additional_data":false},{"leaderboard":"/sota/speech-emotion-recognition-on-resd","task":"Speech Emotion Recognition","dataset":"RESD","model":"emotion2vec+large","rank_in_archive_order":2,"of":3,"metrics":{"Unweighted Accuracy (UA)":"69.1","Weighted Accuracy (WA)":"69.5","Weighted F1":"68.8"},"uses_additional_data":false},{"leaderboard":"/sota/speech-emotion-recognition-on-resd","task":"Speech Emotion Recognition","dataset":"RESD","model":"emotion2vec","rank_in_archive_order":3,"of":3,"metrics":{"Unweighted Accuracy (UA)":"65.04","Weighted Accuracy (WA)":"64.75","Weighted F1":"64.53"},"uses_additional_data":false}],"syntology":{"atlas_url":"https://app.syntology.ai/?focus=2312.15185","mcp":{"get_harvested_code_for_paper":{"arxiv_id":"2312.15185"}},"developers":"https://syntology.ai/developers","read_at":"2026-09-24T18:15:14+00:00","read_at_is":"when the build read Syntology's graph, not when any sample ran","claim":"Per-sample execution status on synthesized fixtures; not a correctness claim about the paper. Samples come from repositories linked to the paper, official or community; repo_kind says which.","repos":[{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/alibaba-damo-academy/FunASR","reach":{"status":"ok","spdx":"MIT"}},{"provenance":"external:paperswithcode_snapshot_2025-07-28","url":"https://github.com/ddlBoJack/emotion2vec","reach":{"status":"ok"}}],"summary":{"ran":8,"unverified":12},"by_repo_kind":{"official":{"samples":9,"ran":8,"repositories":1},"listed":{"samples":11,"ran":0,"repositories":1}},"repo_kind_vocabulary":{"official":"The archive marks this repository official for the paper","named_in_paper":"The archive records that the paper mentions this repository; it is not marked official","listed":"In the archive's code links for this paper, not marked official and not recorded as mentioned in the paper","found_in_text":"Syntology found this repository in the paper's own text; whether it is the authors' implementation is not asserted","community":"Not in the archive's code links for this paper; a community repository Syntology harvested"},"n_pointer_only_for_licence":9,"samples":[{"code_sha256_prefix":"e2766c2b91f8621c","entry":"compute_unweighted_accuracy","repo":"ddlBoJack/emotion2vec","repo_kind":"official","path":"iemocap_downstream/utils.py","file_url":"https://github.com/ddlBoJack/emotion2vec/blob/HEAD/iemocap_downstream/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":true,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"e2766c2b91f8621c"}},{"code_sha256_prefix":"5d5be3c7916a67cb","entry":"get_annealed_rate","repo":"ddlBoJack/emotion2vec","repo_kind":"official","path":"upstream/models/base.py","file_url":"https://github.com/ddlBoJack/emotion2vec/blob/HEAD/upstream/models/base.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"5d5be3c7916a67cb"}},{"code_sha256_prefix":"c032cac6cdbab804","entry":"load_dataset","repo":"ddlBoJack/emotion2vec","repo_kind":"official","path":"iemocap_downstream/data.py","file_url":"https://github.com/ddlBoJack/emotion2vec/blob/HEAD/iemocap_downstream/data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"c032cac6cdbab804"}},{"code_sha256_prefix":"54b7c05f4b14a478","entry":"load_ssl_features","repo":"ddlBoJack/emotion2vec","repo_kind":"official","path":"iemocap_downstream/data.py","file_url":"https://github.com/ddlBoJack/emotion2vec/blob/HEAD/iemocap_downstream/data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"54b7c05f4b14a478"}},{"code_sha256_prefix":"ee9ac852b4eb7975","entry":"random_masking","repo":"ddlBoJack/emotion2vec","repo_kind":"official","path":"upstream/models/base.py","file_url":"https://github.com/ddlBoJack/emotion2vec/blob/HEAD/upstream/models/base.py","link_basis":"harvester_set","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"ee9ac852b4eb7975"}},{"code_sha256_prefix":"3f0a8af8defa1699","entry":"train_one_epoch","repo":"ddlBoJack/emotion2vec","repo_kind":"official","path":"iemocap_downstream/utils.py","file_url":"https://github.com/ddlBoJack/emotion2vec/blob/HEAD/iemocap_downstream/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3f0a8af8defa1699"}},{"code_sha256_prefix":"7707e62c49681b18","entry":"train_valid_test_iemocap_dataloader","repo":"ddlBoJack/emotion2vec","repo_kind":"official","path":"iemocap_downstream/data.py","file_url":"https://github.com/ddlBoJack/emotion2vec/blob/HEAD/iemocap_downstream/data.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"7707e62c49681b18"}},{"code_sha256_prefix":"9d77c0bc89a728ef","entry":"validate_and_test","repo":"ddlBoJack/emotion2vec","repo_kind":"official","path":"iemocap_downstream/utils.py","file_url":"https://github.com/ddlBoJack/emotion2vec/blob/HEAD/iemocap_downstream/utils.py","link_basis":"first_harvest_node","language":"python","status":"ran","verification_level":1,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"9d77c0bc89a728ef"}},{"code_sha256_prefix":"e736efd2088f5330","entry":"add_file_root_path","repo":"alibaba-damo-academy/FunASR","repo_kind":"listed","path":"funasr/download/download_model_from_hub.py","file_url":"https://github.com/alibaba-damo-academy/FunASR/blob/HEAD/funasr/download/download_model_from_hub.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e736efd2088f5330"}},{"code_sha256_prefix":"d94882bbd36f686d","entry":"check_vllm_applicable","repo":"alibaba-damo-academy/FunASR","repo_kind":"listed","path":"funasr/auto/auto_model_vllm.py","file_url":"https://github.com/alibaba-damo-academy/FunASR/blob/HEAD/funasr/auto/auto_model_vllm.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d94882bbd36f686d"}},{"code_sha256_prefix":"6c9da623e97fa4fe","entry":"cr_ctc_loss","repo":"alibaba-damo-academy/FunASR","repo_kind":"listed","path":"funasr/losses/cr_ctc.py","file_url":"https://github.com/alibaba-damo-academy/FunASR/blob/HEAD/funasr/losses/cr_ctc.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"6c9da623e97fa4fe"}},{"code_sha256_prefix":"3a95f65cb24c84e6","entry":"gather_unmasked","repo":"ddlBoJack/emotion2vec","repo_kind":"official","path":"upstream/models/base.py","file_url":"https://github.com/ddlBoJack/emotion2vec/blob/HEAD/upstream/models/base.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"NONE","inline_ok":false,"mcp_get_code":{"code_sha256":"3a95f65cb24c84e6"}},{"code_sha256_prefix":"e40f7b6dc5c3977a","entry":"get_human_readable_count","repo":"alibaba-damo-academy/FunASR","repo_kind":"listed","path":"funasr/train_utils/model_summary.py","file_url":"https://github.com/alibaba-damo-academy/FunASR/blob/HEAD/funasr/train_utils/model_summary.py","link_basis":"harvester_set","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"e40f7b6dc5c3977a"}},{"code_sha256_prefix":"5288c864dbd0538d","entry":"get_or_download_model_dir","repo":"alibaba-damo-academy/FunASR","repo_kind":"listed","path":"funasr/download/download_model_from_hub.py","file_url":"https://github.com/alibaba-damo-academy/FunASR/blob/HEAD/funasr/download/download_model_from_hub.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"5288c864dbd0538d"}},{"code_sha256_prefix":"1c80497412c69732","entry":"get_or_download_model_dir_hf","repo":"alibaba-damo-academy/FunASR","repo_kind":"listed","path":"funasr/download/download_model_from_hub.py","file_url":"https://github.com/alibaba-damo-academy/FunASR/blob/HEAD/funasr/download/download_model_from_hub.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"1c80497412c69732"}},{"code_sha256_prefix":"d6d4ea8b2806c520","entry":"install_requirements","repo":"alibaba-damo-academy/FunASR","repo_kind":"listed","path":"funasr/utils/install_model_requirements.py","file_url":"https://github.com/alibaba-damo-academy/FunASR/blob/HEAD/funasr/utils/install_model_requirements.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"d6d4ea8b2806c520"}},{"code_sha256_prefix":"26200c7d0299db96","entry":"model_summary","repo":"alibaba-damo-academy/FunASR","repo_kind":"listed","path":"funasr/train_utils/model_summary.py","file_url":"https://github.com/alibaba-damo-academy/FunASR/blob/HEAD/funasr/train_utils/model_summary.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"26200c7d0299db96"}},{"code_sha256_prefix":"229df00957f1445e","entry":"pip_install_r","repo":"alibaba-damo-academy/FunASR","repo_kind":"listed","path":"funasr/utils/install_model_requirements.py","file_url":"https://github.com/alibaba-damo-academy/FunASR/blob/HEAD/funasr/utils/install_model_requirements.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"229df00957f1445e"}},{"code_sha256_prefix":"fc2c2ae72cabe336","entry":"prepare_vllm_weights","repo":"alibaba-damo-academy/FunASR","repo_kind":"listed","path":"funasr/auto/auto_model_vllm.py","file_url":"https://github.com/alibaba-damo-academy/FunASR/blob/HEAD/funasr/auto/auto_model_vllm.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"fc2c2ae72cabe336"}},{"code_sha256_prefix":"54138299a3e55e00","entry":"to_bytes","repo":"alibaba-damo-academy/FunASR","repo_kind":"listed","path":"funasr/train_utils/model_summary.py","file_url":"https://github.com/alibaba-damo-academy/FunASR/blob/HEAD/funasr/train_utils/model_summary.py","link_basis":"plan_row","language":"python","status":"unverified","verification_level":0,"contract_check":null,"metamorphic_tier":null,"behaviour_fingerprint":false,"licence":"MIT","inline_ok":true,"mcp_get_code":{"code_sha256":"54138299a3e55e00"}}]},"arxiv_metadata":null,"syntology_extracted_results":null}